gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2018 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V4_REGNUM          36)
  61     (V8_REGNUM          40)
  62     (V12_REGNUM         44)
  63     (V15_REGNUM         47)
  64     (V16_REGNUM         48)
  65     (V20_REGNUM         52)
  66     (V24_REGNUM         56)
  67     (V28_REGNUM         60)
  68     (V31_REGNUM         63)
  69     (LAST_SAVED_REGNUM  63)
  70     (SFP_REGNUM         64)
  71     (AP_REGNUM          65)
  72     (CC_REGNUM          66)
  73     ;; Defined only to make the DWARF description simpler.
  74     (VG_REGNUM          67)
  75     (P0_REGNUM          68)
  76     (P1_REGNUM          69)
  77     (P2_REGNUM          70)
  78     (P3_REGNUM          71)
  79     (P4_REGNUM          72)
  80     (P5_REGNUM          73)
  81     (P6_REGNUM          74)
  82     (P7_REGNUM          75)
  83     (P8_REGNUM          76)
  84     (P9_REGNUM          77)
  85     (P10_REGNUM         78)
  86     (P11_REGNUM         79)
  87     (P12_REGNUM         80)
  88     (P13_REGNUM         81)
  89     (P14_REGNUM         82)
  90     (P15_REGNUM         83)
  91     ;; A couple of call-clobbered registers that we need to reserve when
  92     ;; tracking speculation this is not ABI, so is subject to change.
  93     (SPECULATION_TRACKER_REGNUM 15)
  94     (SPECULATION_SCRATCH_REGNUM 14)
  95   ]
  96 )
  97
  98 (define_c_enum "unspec" [
  99     UNSPEC_AUTI1716
 100     UNSPEC_AUTISP
 101     UNSPEC_CASESI
 102     UNSPEC_CRC32B
 103     UNSPEC_CRC32CB
 104     UNSPEC_CRC32CH
 105     UNSPEC_CRC32CW
 106     UNSPEC_CRC32CX
 107     UNSPEC_CRC32H
 108     UNSPEC_CRC32W
 109     UNSPEC_CRC32X
 110     UNSPEC_FCVTZS
 111     UNSPEC_FCVTZU
 112     UNSPEC_URECPE
 113     UNSPEC_FRECPE
 114     UNSPEC_FRECPS
 115     UNSPEC_FRECPX
 116     UNSPEC_FRINTA
 117     UNSPEC_FRINTI
 118     UNSPEC_FRINTM
 119     UNSPEC_FRINTN
 120     UNSPEC_FRINTP
 121     UNSPEC_FRINTX
 122     UNSPEC_FRINTZ
 123     UNSPEC_GOTSMALLPIC
 124     UNSPEC_GOTSMALLPIC28K
 125     UNSPEC_GOTSMALLTLS
 126     UNSPEC_GOTTINYPIC
 127     UNSPEC_GOTTINYTLS
 128     UNSPEC_LD1
 129     UNSPEC_LD2
 130     UNSPEC_LD2_DREG
 131     UNSPEC_LD2_DUP
 132     UNSPEC_LD3
 133     UNSPEC_LD3_DREG
 134     UNSPEC_LD3_DUP
 135     UNSPEC_LD4
 136     UNSPEC_LD4_DREG
 137     UNSPEC_LD4_DUP
 138     UNSPEC_LD2_LANE
 139     UNSPEC_LD3_LANE
 140     UNSPEC_LD4_LANE
 141     UNSPEC_MB
 142     UNSPEC_NOP
 143     UNSPEC_PACI1716
 144     UNSPEC_PACISP
 145     UNSPEC_PRLG_STK
 146     UNSPEC_REV
 147     UNSPEC_RBIT
 148     UNSPEC_SABAL
 149     UNSPEC_SABDL2
 150     UNSPEC_SADALP
 151     UNSPEC_SCVTF
 152     UNSPEC_SISD_NEG
 153     UNSPEC_SISD_SSHL
 154     UNSPEC_SISD_USHL
 155     UNSPEC_SSHL_2S
 156     UNSPEC_ST1
 157     UNSPEC_ST2
 158     UNSPEC_ST3
 159     UNSPEC_ST4
 160     UNSPEC_ST2_LANE
 161     UNSPEC_ST3_LANE
 162     UNSPEC_ST4_LANE
 163     UNSPEC_TLS
 164     UNSPEC_TLSDESC
 165     UNSPEC_TLSLE12
 166     UNSPEC_TLSLE24
 167     UNSPEC_TLSLE32
 168     UNSPEC_TLSLE48
 169     UNSPEC_UABAL
 170     UNSPEC_UABDL2
 171     UNSPEC_UADALP
 172     UNSPEC_UCVTF
 173     UNSPEC_USHL_2S
 174     UNSPEC_VSTRUCTDUMMY
 175     UNSPEC_SP_SET
 176     UNSPEC_SP_TEST
 177     UNSPEC_RSQRT
 178     UNSPEC_RSQRTE
 179     UNSPEC_RSQRTS
 180     UNSPEC_NZCV
 181     UNSPEC_XPACLRI
 182     UNSPEC_LD1_SVE
 183     UNSPEC_ST1_SVE
 184     UNSPEC_LD1RQ
 185     UNSPEC_LD1_GATHER
 186     UNSPEC_ST1_SCATTER
 187     UNSPEC_MERGE_PTRUE
 188     UNSPEC_PTEST_PTRUE
 189     UNSPEC_UNPACKSHI
 190     UNSPEC_UNPACKUHI
 191     UNSPEC_UNPACKSLO
 192     UNSPEC_UNPACKULO
 193     UNSPEC_PACK
 194     UNSPEC_FLOAT_CONVERT
 195     UNSPEC_WHILE_LO
 196     UNSPEC_LDN
 197     UNSPEC_STN
 198     UNSPEC_INSR
 199     UNSPEC_CLASTB
 200     UNSPEC_FADDA
 201     UNSPEC_REV_SUBREG
 202     UNSPEC_SPECULATION_TRACKER
 203 ])
 204
 205 (define_c_enum "unspecv" [
 206     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 207     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 208     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 209     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 210     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 211     UNSPECV_BLOCKAGE            ; Represent a blockage
 212     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 213     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 214   ]
 215 )
 216
 217 ;; If further include files are added the defintion of MD_INCLUDES
 218 ;; must be updated.
 219
 220 (include "constraints.md")
 221 (include "predicates.md")
 222 (include "iterators.md")
 223
 224 ;; -------------------------------------------------------------------
 225 ;; Instruction types and attributes
 226 ;; -------------------------------------------------------------------
 227
 228 ; The "type" attribute is included here from AArch32 backend to be able
 229 ; to share pipeline descriptions.
 230 (include "../arm/types.md")
 231
 232 ;; It is important to set the fp or simd attributes to yes when a pattern
 233 ;; alternative uses the FP or SIMD register files, usually signified by use of
 234 ;; the 'w' constraint.  This will ensure that the alternative will be
 235 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 236 ;; architecture extensions.  If all the alternatives in a pattern use the
 237 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 238 ;; or TARGET_SIMD.
 239
 240 ;; Attribute that specifies whether or not the instruction touches fp
 241 ;; registers.  When this is set to yes for an alternative, that alternative
 242 ;; will be disabled when !TARGET_FLOAT.
 243 (define_attr "fp" "no,yes" (const_string "no"))
 244
 245 ;; Attribute that specifies whether or not the instruction touches half
 246 ;; precision fp registers.  When this is set to yes for an alternative,
 247 ;; that alternative will be disabled when !TARGET_FP_F16INST.
 248 (define_attr "fp16" "no,yes" (const_string "no"))
 249
 250 ;; Attribute that specifies whether or not the instruction touches simd
 251 ;; registers.  When this is set to yes for an alternative, that alternative
 252 ;; will be disabled when !TARGET_SIMD.
 253 (define_attr "simd" "no,yes" (const_string "no"))
 254
 255 ;; Attribute that specifies whether or not the instruction uses SVE.
 256 ;; When this is set to yes for an alternative, that alternative
 257 ;; will be disabled when !TARGET_SVE.
 258 (define_attr "sve" "no,yes" (const_string "no"))
 259
 260 ;; Attribute that controls whether an alternative is enabled or not.
 261 ;; Currently it is only used to disable alternatives which touch fp or simd
 262 ;; registers when -mgeneral-regs-only is specified.
 263 (define_attr "enabled" "no,yes"
 264   (cond [(ior
 265             (and (eq_attr "fp" "yes")
 266                  (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
 267             (and (eq_attr "simd" "yes")
 268                  (eq (symbol_ref "TARGET_SIMD") (const_int 0)))
 269             (and (eq_attr "fp16" "yes")
 270                  (eq (symbol_ref "TARGET_FP_F16INST") (const_int 0)))
 271             (and (eq_attr "sve" "yes")
 272                  (eq (symbol_ref "TARGET_SVE") (const_int 0))))
 273             (const_string "no")
 274         ] (const_string "yes")))
 275
 276 ;; Attribute that specifies whether we are dealing with a branch to a
 277 ;; label that is far away, i.e. further away than the maximum/minimum
 278 ;; representable in a signed 21-bits number.
 279 ;; 0 :=: no
 280 ;; 1 :=: yes
 281 (define_attr "far_branch" "" (const_int 0))
 282
 283 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 284 (define_attr "movprfx" "no,yes" (const_string "no"))
 285
 286 (define_attr "length" ""
 287   (cond [(eq_attr "movprfx" "yes")
 288            (const_int 8)
 289         ] (const_int 4)))
 290
 291 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 292 ;; no predicated insns.
 293 (define_attr "predicated" "yes,no" (const_string "no"))
 294
 295 ;; Set to true on an insn that requires the speculation tracking state to be
 296 ;; in the tracking register before the insn issues.  Otherwise the compiler
 297 ;; may chose to hold the tracking state encoded in SP.
 298 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 299
 300 ;; -------------------------------------------------------------------
 301 ;; Pipeline descriptions and scheduling
 302 ;; -------------------------------------------------------------------
 303
 304 ;; Processor types.
 305 (include "aarch64-tune.md")
 306
 307 ;; Scheduling
 308 (include "../arm/cortex-a53.md")
 309 (include "../arm/cortex-a57.md")
 310 (include "../arm/exynos-m1.md")
 311 (include "falkor.md")
 312 (include "thunderx.md")
 313 (include "../arm/xgene1.md")
 314 (include "thunderx2t99.md")
 315
 316 ;; -------------------------------------------------------------------
 317 ;; Jumps and other miscellaneous insns
 318 ;; -------------------------------------------------------------------
 319
 320 (define_insn "indirect_jump"
 321   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 322   ""
 323   "br\\t%0"
 324   [(set_attr "type" "branch")]
 325 )
 326
 327 (define_insn "jump"
 328   [(set (pc) (label_ref (match_operand 0 "" "")))]
 329   ""
 330   "b\\t%l0"
 331   [(set_attr "type" "branch")]
 332 )
 333
 334 (define_expand "cbranch<mode>4"
 335   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 336                             [(match_operand:GPI 1 "register_operand" "")
 337                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 338                            (label_ref (match_operand 3 "" ""))
 339                            (pc)))]
 340   ""
 341   "
 342   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 343                                          operands[2]);
 344   operands[2] = const0_rtx;
 345   "
 346 )
 347
 348 (define_expand "cbranch<mode>4"
 349   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 350                             [(match_operand:GPF 1 "register_operand" "")
 351                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 352                            (label_ref (match_operand 3 "" ""))
 353                            (pc)))]
 354   ""
 355   "
 356   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 357                                          operands[2]);
 358   operands[2] = const0_rtx;
 359   "
 360 )
 361
 362 (define_expand "cbranchcc4"
 363   [(set (pc) (if_then_else
 364               (match_operator 0 "aarch64_comparison_operator"
 365                [(match_operand 1 "cc_register" "")
 366                 (match_operand 2 "const0_operand")])
 367               (label_ref (match_operand 3 "" ""))
 368               (pc)))]
 369   ""
 370   "")
 371
 372 (define_insn "ccmp<mode>"
 373   [(set (match_operand:CC 1 "cc_register" "")
 374         (if_then_else:CC
 375           (match_operator 4 "aarch64_comparison_operator"
 376            [(match_operand 0 "cc_register" "")
 377             (const_int 0)])
 378           (compare:CC
 379             (match_operand:GPI 2 "register_operand" "r,r,r")
 380             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 381           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 382   ""
 383   "@
 384    ccmp\\t%<w>2, %<w>3, %k5, %m4
 385    ccmp\\t%<w>2, %3, %k5, %m4
 386    ccmn\\t%<w>2, #%n3, %k5, %m4"
 387   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 388 )
 389
 390 (define_insn "fccmp<mode>"
 391   [(set (match_operand:CCFP 1 "cc_register" "")
 392         (if_then_else:CCFP
 393           (match_operator 4 "aarch64_comparison_operator"
 394            [(match_operand 0 "cc_register" "")
 395             (const_int 0)])
 396           (compare:CCFP
 397             (match_operand:GPF 2 "register_operand" "w")
 398             (match_operand:GPF 3 "register_operand" "w"))
 399           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 400   "TARGET_FLOAT"
 401   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 402   [(set_attr "type" "fccmp<s>")]
 403 )
 404
 405 (define_insn "fccmpe<mode>"
 406   [(set (match_operand:CCFPE 1 "cc_register" "")
 407          (if_then_else:CCFPE
 408           (match_operator 4 "aarch64_comparison_operator"
 409            [(match_operand 0 "cc_register" "")
 410           (const_int 0)])
 411            (compare:CCFPE
 412             (match_operand:GPF 2 "register_operand" "w")
 413             (match_operand:GPF 3 "register_operand" "w"))
 414           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 415   "TARGET_FLOAT"
 416   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 417   [(set_attr "type" "fccmp<s>")]
 418 )
 419
 420 ;; Expansion of signed mod by a power of 2 using CSNEG.
 421 ;; For x0 % n where n is a power of 2 produce:
 422 ;; negs   x1, x0
 423 ;; and    x0, x0, #(n - 1)
 424 ;; and    x1, x1, #(n - 1)
 425 ;; csneg  x0, x0, x1, mi
 426
 427 (define_expand "mod<mode>3"
 428   [(match_operand:GPI 0 "register_operand" "")
 429    (match_operand:GPI 1 "register_operand" "")
 430    (match_operand:GPI 2 "const_int_operand" "")]
 431   ""
 432   {
 433     HOST_WIDE_INT val = INTVAL (operands[2]);
 434
 435     if (val <= 0
 436        || exact_log2 (val) <= 0
 437        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 438       FAIL;
 439
 440     rtx mask = GEN_INT (val - 1);
 441
 442     /* In the special case of x0 % 2 we can do the even shorter:
 443         cmp    x0, xzr
 444         and    x0, x0, 1
 445         cneg   x0, x0, lt.  */
 446     if (val == 2)
 447       {
 448         rtx masked = gen_reg_rtx (<MODE>mode);
 449         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 450         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 451         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 452         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 453         DONE;
 454       }
 455
 456     rtx neg_op = gen_reg_rtx (<MODE>mode);
 457     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 458
 459     /* Extract the condition register and mode.  */
 460     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 461     rtx cc_reg = SET_DEST (cmp);
 462     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 463
 464     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 465     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 466
 467     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 468     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 469
 470     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 471                                        masked_neg, masked_pos));
 472     DONE;
 473   }
 474 )
 475
 476 (define_insn "condjump"
 477   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 478                             [(match_operand 1 "cc_register" "") (const_int 0)])
 479                            (label_ref (match_operand 2 "" ""))
 480                            (pc)))]
 481   ""
 482   {
 483     if (get_attr_length (insn) == 8)
 484       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 485     else
 486       return  "b%m0\\t%l2";
 487   }
 488   [(set_attr "type" "branch")
 489    (set (attr "length")
 490         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 491                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 492                       (const_int 4)
 493                       (const_int 8)))
 494    (set (attr "far_branch")
 495         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 496                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 497                       (const_int 0)
 498                       (const_int 1)))]
 499 )
 500
 501 ;; For a 24-bit immediate CST we can optimize the compare for equality
 502 ;; and branch sequence from:
 503 ;;      mov     x0, #imm1
 504 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 505 ;;      cmp     x1, x0
 506 ;;      b<ne,eq> .Label
 507 ;; into the shorter:
 508 ;;      sub     x0, x1, #(CST & 0xfff000)
 509 ;;      subs    x0, x0, #(CST & 0x000fff)
 510 ;;      b<ne,eq> .Label
 511 (define_insn_and_split "*compare_condjump<mode>"
 512   [(set (pc) (if_then_else (EQL
 513                               (match_operand:GPI 0 "register_operand" "r")
 514                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 515                            (label_ref:P (match_operand 2 "" ""))
 516                            (pc)))]
 517   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 518    && !aarch64_plus_operand (operands[1], <MODE>mode)
 519    && !reload_completed"
 520   "#"
 521   "&& true"
 522   [(const_int 0)]
 523   {
 524     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 525     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 526     rtx tmp = gen_reg_rtx (<MODE>mode);
 527     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 528     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 529     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 530     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 531     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 532     DONE;
 533   }
 534 )
 535
 536 (define_expand "casesi"
 537   [(match_operand:SI 0 "register_operand" "")   ; Index
 538    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 539    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 540    (match_operand:DI 3 "" "")                   ; Table label
 541    (match_operand:DI 4 "" "")]                  ; Out of range label
 542   ""
 543   {
 544     if (operands[1] != const0_rtx)
 545       {
 546         rtx reg = gen_reg_rtx (SImode);
 547
 548         /* Canonical RTL says that if you have:
 549
 550            (minus (X) (CONST))
 551
 552            then this should be emitted as:
 553
 554            (plus (X) (-CONST))
 555
 556            The use of trunc_int_for_mode ensures that the resulting
 557            constant can be represented in SImode, this is important
 558            for the corner case where operand[1] is INT_MIN.  */
 559
 560         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 561
 562         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 563               (operands[1], SImode))
 564           operands[1] = force_reg (SImode, operands[1]);
 565         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 566         operands[0] = reg;
 567       }
 568
 569     if (!aarch64_plus_operand (operands[2], SImode))
 570       operands[2] = force_reg (SImode, operands[2]);
 571     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 572                                                  const0_rtx),
 573                                     operands[0], operands[2], operands[4]));
 574
 575     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 576     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 577                                          operands[3]));
 578     DONE;
 579   }
 580 )
 581
 582 (define_insn "casesi_dispatch"
 583   [(parallel
 584     [(set (pc)
 585           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 586                            (match_operand:SI 1 "register_operand" "r")]
 587                         UNSPEC_CASESI)))
 588      (clobber (reg:CC CC_REGNUM))
 589      (clobber (match_scratch:DI 3 "=r"))
 590      (clobber (match_scratch:DI 4 "=r"))
 591      (use (label_ref (match_operand 2 "" "")))])]
 592   ""
 593   "*
 594   return aarch64_output_casesi (operands);
 595   "
 596   [(set_attr "length" "16")
 597    (set_attr "type" "branch")]
 598 )
 599
 600 (define_insn "nop"
 601   [(unspec[(const_int 0)] UNSPEC_NOP)]
 602   ""
 603   "nop"
 604   [(set_attr "type" "no_insn")]
 605 )
 606
 607 (define_insn "prefetch"
 608   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 609             (match_operand:QI 1 "const_int_operand" "")
 610             (match_operand:QI 2 "const_int_operand" ""))]
 611   ""
 612   {
 613     const char * pftype[2][4] =
 614     {
 615       {"prfm\\tPLDL1STRM, %0",
 616        "prfm\\tPLDL3KEEP, %0",
 617        "prfm\\tPLDL2KEEP, %0",
 618        "prfm\\tPLDL1KEEP, %0"},
 619       {"prfm\\tPSTL1STRM, %0",
 620        "prfm\\tPSTL3KEEP, %0",
 621        "prfm\\tPSTL2KEEP, %0",
 622        "prfm\\tPSTL1KEEP, %0"},
 623     };
 624
 625     int locality = INTVAL (operands[2]);
 626
 627     gcc_assert (IN_RANGE (locality, 0, 3));
 628
 629     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 630        the address into a DImode MEM so that aarch64_print_operand knows
 631        how to print it.  */
 632     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 633     return pftype[INTVAL(operands[1])][locality];
 634   }
 635   [(set_attr "type" "load_4")]
 636 )
 637
 638 (define_insn "trap"
 639   [(trap_if (const_int 1) (const_int 8))]
 640   ""
 641   "brk #1000"
 642   [(set_attr "type" "trap")])
 643
 644 (define_expand "prologue"
 645   [(clobber (const_int 0))]
 646   ""
 647   "
 648   aarch64_expand_prologue ();
 649   DONE;
 650   "
 651 )
 652
 653 (define_expand "epilogue"
 654   [(clobber (const_int 0))]
 655   ""
 656   "
 657   aarch64_expand_epilogue (false);
 658   DONE;
 659   "
 660 )
 661
 662 (define_expand "sibcall_epilogue"
 663   [(clobber (const_int 0))]
 664   ""
 665   "
 666   aarch64_expand_epilogue (true);
 667   DONE;
 668   "
 669 )
 670
 671 (define_insn "*do_return"
 672   [(return)]
 673   ""
 674   {
 675     if (aarch64_return_address_signing_enabled ()
 676         && TARGET_ARMV8_3
 677         && !crtl->calls_eh_return)
 678       return "retaa";
 679
 680     return "ret";
 681   }
 682   [(set_attr "type" "branch")]
 683 )
 684
 685 (define_expand "return"
 686   [(simple_return)]
 687   "aarch64_use_return_insn_p ()"
 688   ""
 689 )
 690
 691 (define_insn "simple_return"
 692   [(simple_return)]
 693   ""
 694   "ret"
 695   [(set_attr "type" "branch")]
 696 )
 697
 698 (define_insn "*cb<optab><mode>1"
 699   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 700                                 (const_int 0))
 701                            (label_ref (match_operand 1 "" ""))
 702                            (pc)))]
 703   "!aarch64_track_speculation"
 704   {
 705     if (get_attr_length (insn) == 8)
 706       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 707     else
 708       return "<cbz>\\t%<w>0, %l1";
 709   }
 710   [(set_attr "type" "branch")
 711    (set (attr "length")
 712         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 713                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 714                       (const_int 4)
 715                       (const_int 8)))
 716    (set (attr "far_branch")
 717         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 718                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 719                       (const_int 0)
 720                       (const_int 1)))]
 721 )
 722
 723 (define_insn "*tb<optab><mode>1"
 724   [(set (pc) (if_then_else
 725               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 726                                     (const_int 1)
 727                                     (match_operand 1
 728                                       "aarch64_simd_shift_imm_<mode>" "n"))
 729                    (const_int 0))
 730              (label_ref (match_operand 2 "" ""))
 731              (pc)))
 732    (clobber (reg:CC CC_REGNUM))]
 733   "!aarch64_track_speculation"
 734   {
 735     if (get_attr_length (insn) == 8)
 736       {
 737         if (get_attr_far_branch (insn) == 1)
 738           return aarch64_gen_far_branch (operands, 2, "Ltb",
 739                                          "<inv_tb>\\t%<w>0, %1, ");
 740         else
 741           {
 742             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 743             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 744           }
 745       }
 746     else
 747       return "<tbz>\t%<w>0, %1, %l2";
 748   }
 749   [(set_attr "type" "branch")
 750    (set (attr "length")
 751         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 752                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 753                       (const_int 4)
 754                       (const_int 8)))
 755    (set (attr "far_branch")
 756         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 757                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 758                       (const_int 0)
 759                       (const_int 1)))]
 760
 761 )
 762
 763 (define_insn "*cb<optab><mode>1"
 764   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 765                                  (const_int 0))
 766                            (label_ref (match_operand 1 "" ""))
 767                            (pc)))
 768    (clobber (reg:CC CC_REGNUM))]
 769   "!aarch64_track_speculation"
 770   {
 771     if (get_attr_length (insn) == 8)
 772       {
 773         if (get_attr_far_branch (insn) == 1)
 774           return aarch64_gen_far_branch (operands, 1, "Ltb",
 775                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 776         else
 777           {
 778             char buf[64];
 779             uint64_t val = ((uint64_t) 1)
 780                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 781             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 782             output_asm_insn (buf, operands);
 783             return "<bcond>\t%l1";
 784           }
 785       }
 786     else
 787       return "<tbz>\t%<w>0, <sizem1>, %l1";
 788   }
 789   [(set_attr "type" "branch")
 790    (set (attr "length")
 791         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 792                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 793                       (const_int 4)
 794                       (const_int 8)))
 795    (set (attr "far_branch")
 796         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 797                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 798                       (const_int 0)
 799                       (const_int 1)))]
 800 )
 801
 802 ;; -------------------------------------------------------------------
 803 ;; Subroutine calls and sibcalls
 804 ;; -------------------------------------------------------------------
 805
 806 (define_expand "call"
 807   [(parallel [(call (match_operand 0 "memory_operand" "")
 808                     (match_operand 1 "general_operand" ""))
 809               (use (match_operand 2 "" ""))
 810               (clobber (reg:DI LR_REGNUM))])]
 811   ""
 812   "
 813   {
 814     aarch64_expand_call (NULL_RTX, operands[0], false);
 815     DONE;
 816   }"
 817 )
 818
 819 (define_insn "*call_insn"
 820   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 821          (match_operand 1 "" ""))
 822    (clobber (reg:DI LR_REGNUM))]
 823   ""
 824   "@
 825   blr\\t%0
 826   bl\\t%c0"
 827   [(set_attr "type" "call, call")]
 828 )
 829
 830 (define_expand "call_value"
 831   [(parallel [(set (match_operand 0 "" "")
 832                    (call (match_operand 1 "memory_operand" "")
 833                          (match_operand 2 "general_operand" "")))
 834               (use (match_operand 3 "" ""))
 835               (clobber (reg:DI LR_REGNUM))])]
 836   ""
 837   "
 838   {
 839     aarch64_expand_call (operands[0], operands[1], false);
 840     DONE;
 841   }"
 842 )
 843
 844 (define_insn "*call_value_insn"
 845   [(set (match_operand 0 "" "")
 846         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 847                       (match_operand 2 "" "")))
 848    (clobber (reg:DI LR_REGNUM))]
 849   ""
 850   "@
 851   blr\\t%1
 852   bl\\t%c1"
 853   [(set_attr "type" "call, call")]
 854 )
 855
 856 (define_expand "sibcall"
 857   [(parallel [(call (match_operand 0 "memory_operand" "")
 858                     (match_operand 1 "general_operand" ""))
 859               (return)
 860               (use (match_operand 2 "" ""))])]
 861   ""
 862   {
 863     aarch64_expand_call (NULL_RTX, operands[0], true);
 864     DONE;
 865   }
 866 )
 867
 868 (define_expand "sibcall_value"
 869   [(parallel [(set (match_operand 0 "" "")
 870                    (call (match_operand 1 "memory_operand" "")
 871                          (match_operand 2 "general_operand" "")))
 872               (return)
 873               (use (match_operand 3 "" ""))])]
 874   ""
 875   {
 876     aarch64_expand_call (operands[0], operands[1], true);
 877     DONE;
 878   }
 879 )
 880
 881 (define_insn "*sibcall_insn"
 882   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 883          (match_operand 1 "" ""))
 884    (return)]
 885   "SIBLING_CALL_P (insn)"
 886   "@
 887    br\\t%0
 888    b\\t%c0"
 889   [(set_attr "type" "branch, branch")]
 890 )
 891
 892 (define_insn "*sibcall_value_insn"
 893   [(set (match_operand 0 "" "")
 894         (call (mem:DI
 895                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 896               (match_operand 2 "" "")))
 897    (return)]
 898   "SIBLING_CALL_P (insn)"
 899   "@
 900    br\\t%1
 901    b\\t%c1"
 902   [(set_attr "type" "branch, branch")]
 903 )
 904
 905 ;; Call subroutine returning any type.
 906
 907 (define_expand "untyped_call"
 908   [(parallel [(call (match_operand 0 "")
 909                     (const_int 0))
 910               (match_operand 1 "")
 911               (match_operand 2 "")])]
 912   ""
 913 {
 914   int i;
 915
 916   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 917
 918   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 919     {
 920       rtx set = XVECEXP (operands[2], 0, i);
 921       emit_move_insn (SET_DEST (set), SET_SRC (set));
 922     }
 923
 924   /* The optimizer does not know that the call sets the function value
 925      registers we stored in the result block.  We avoid problems by
 926      claiming that all hard registers are used and clobbered at this
 927      point.  */
 928   emit_insn (gen_blockage ());
 929   DONE;
 930 })
 931
 932 ;; -------------------------------------------------------------------
 933 ;; Moves
 934 ;; -------------------------------------------------------------------
 935
 936 (define_expand "mov<mode>"
 937   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 938         (match_operand:SHORT 1 "general_operand" ""))]
 939   ""
 940   "
 941     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 942       operands[1] = force_reg (<MODE>mode, operands[1]);
 943
 944     if (GET_CODE (operands[1]) == CONST_POLY_INT)
 945       {
 946         aarch64_expand_mov_immediate (operands[0], operands[1]);
 947         DONE;
 948       }
 949   "
 950 )
 951
 952 (define_insn "*mov<mode>_aarch64"
 953   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
 954         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
 955   "(register_operand (operands[0], <MODE>mode)
 956     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 957 {
 958    switch (which_alternative)
 959      {
 960      case 0:
 961        return "mov\t%w0, %w1";
 962      case 1:
 963        return "mov\t%w0, %1";
 964      case 2:
 965        return aarch64_output_scalar_simd_mov_immediate (operands[1],
 966                                                         <MODE>mode);
 967      case 3:
 968        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
 969      case 4:
 970        return "ldr<size>\t%w0, %1";
 971      case 5:
 972        return "ldr\t%<size>0, %1";
 973      case 6:
 974        return "str<size>\t%w1, %0";
 975      case 7:
 976        return "str\t%<size>1, %0";
 977      case 8:
 978        return "umov\t%w0, %1.<v>[0]";
 979      case 9:
 980        return "dup\t%0.<Vallxd>, %w1";
 981      case 10:
 982        return "dup\t%<Vetype>0, %1.<v>[0]";
 983      default:
 984        gcc_unreachable ();
 985      }
 986 }
 987   ;; The "mov_imm" type for CNT is just a placeholder.
 988   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
 989                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
 990    (set_attr "simd" "*,*,yes,*,*,*,*,*,yes,yes,yes")
 991    (set_attr "sve" "*,*,*,yes,*,*,*,*,*,*,*")]
 992 )
 993
 994 (define_expand "mov<mode>"
 995   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
 996         (match_operand:GPI 1 "general_operand" ""))]
 997   ""
 998   "
 999     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1000         && <MODE>mode == DImode
1001         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1002       DONE;
1003
1004     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1005       operands[1] = force_reg (<MODE>mode, operands[1]);
1006
1007     /* FIXME: RR we still need to fix up what we are doing with
1008        symbol_refs and other types of constants.  */
1009     if (CONSTANT_P (operands[1])
1010         && !CONST_INT_P (operands[1]))
1011      {
1012        aarch64_expand_mov_immediate (operands[0], operands[1]);
1013        DONE;
1014      }
1015   "
1016 )
1017
1018 (define_insn_and_split "*movsi_aarch64"
1019   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1020         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1021   "(register_operand (operands[0], SImode)
1022     || aarch64_reg_or_zero (operands[1], SImode))"
1023   "@
1024    mov\\t%w0, %w1
1025    mov\\t%w0, %w1
1026    mov\\t%w0, %w1
1027    mov\\t%w0, %1
1028    #
1029    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1030    ldr\\t%w0, %1
1031    ldr\\t%s0, %1
1032    str\\t%w1, %0
1033    str\\t%s1, %0
1034    adr\\t%x0, %c1
1035    adrp\\t%x0, %A1
1036    fmov\\t%s0, %w1
1037    fmov\\t%w0, %s1
1038    fmov\\t%s0, %s1
1039    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1040   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1041     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1042    [(const_int 0)]
1043    "{
1044        aarch64_expand_mov_immediate (operands[0], operands[1]);
1045        DONE;
1046     }"
1047   ;; The "mov_imm" type for CNT is just a placeholder.
1048   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1049                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1050    (set_attr "fp" "*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1051    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1052    (set_attr "sve" "*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1053 )
1054
1055 (define_insn_and_split "*movdi_aarch64"
1056   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1057         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1058   "(register_operand (operands[0], DImode)
1059     || aarch64_reg_or_zero (operands[1], DImode))"
1060   "@
1061    mov\\t%x0, %x1
1062    mov\\t%0, %x1
1063    mov\\t%x0, %1
1064    mov\\t%x0, %1
1065    mov\\t%w0, %1
1066    #
1067    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1068    ldr\\t%x0, %1
1069    ldr\\t%d0, %1
1070    str\\t%x1, %0
1071    str\\t%d1, %0
1072    adr\\t%x0, %c1
1073    adrp\\t%x0, %A1
1074    fmov\\t%d0, %x1
1075    fmov\\t%x0, %d1
1076    fmov\\t%d0, %d1
1077    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1078    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1079     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1080    [(const_int 0)]
1081    "{
1082        aarch64_expand_mov_immediate (operands[0], operands[1]);
1083        DONE;
1084     }"
1085   ;; The "mov_imm" type for CNTD is just a placeholder.
1086   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1087                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1088                      neon_move")
1089    (set_attr "fp" "*,*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1090    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1091    (set_attr "sve" "*,*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1092 )
1093
1094 (define_insn "insv_imm<mode>"
1095   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1096                           (const_int 16)
1097                           (match_operand:GPI 1 "const_int_operand" "n"))
1098         (match_operand:GPI 2 "const_int_operand" "n"))]
1099   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1100    && UINTVAL (operands[1]) % 16 == 0"
1101   "movk\\t%<w>0, %X2, lsl %1"
1102   [(set_attr "type" "mov_imm")]
1103 )
1104
1105 (define_expand "movti"
1106   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1107         (match_operand:TI 1 "general_operand" ""))]
1108   ""
1109   "
1110     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1111       operands[1] = force_reg (TImode, operands[1]);
1112
1113     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1114       {
1115         emit_move_insn (gen_lowpart (DImode, operands[0]),
1116                         gen_lowpart (DImode, operands[1]));
1117         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1118         DONE;
1119       }
1120   "
1121 )
1122
1123 (define_insn "*movti_aarch64"
1124   [(set (match_operand:TI 0
1125          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1126         (match_operand:TI 1
1127          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1128   "(register_operand (operands[0], TImode)
1129     || aarch64_reg_or_zero (operands[1], TImode))"
1130   "@
1131    #
1132    #
1133    #
1134    mov\\t%0.16b, %1.16b
1135    ldp\\t%0, %H0, %1
1136    stp\\t%1, %H1, %0
1137    stp\\txzr, xzr, %0
1138    ldr\\t%q0, %1
1139    str\\t%q1, %0"
1140   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1141                              load_16,store_16,store_16,\
1142                              load_16,store_16")
1143    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1144    (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1145    (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1146 )
1147
1148 ;; Split a TImode register-register or register-immediate move into
1149 ;; its component DImode pieces, taking care to handle overlapping
1150 ;; source and dest registers.
1151 (define_split
1152    [(set (match_operand:TI 0 "register_operand" "")
1153          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1154   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1155   [(const_int 0)]
1156 {
1157   aarch64_split_128bit_move (operands[0], operands[1]);
1158   DONE;
1159 })
1160
1161 (define_expand "mov<mode>"
1162   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1163         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1164   ""
1165   {
1166     if (!TARGET_FLOAT)
1167       {
1168         aarch64_err_no_fpadvsimd (<MODE>mode);
1169         FAIL;
1170       }
1171
1172     if (GET_CODE (operands[0]) == MEM
1173         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1174               && aarch64_float_const_zero_rtx_p (operands[1])))
1175       operands[1] = force_reg (<MODE>mode, operands[1]);
1176   }
1177 )
1178
1179 (define_insn "*movhf_aarch64"
1180   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1181         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1182   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1183     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1184   "@
1185    movi\\t%0.4h, #0
1186    fmov\\t%h0, %w1
1187    dup\\t%w0.4h, %w1
1188    umov\\t%w0, %1.h[0]
1189    mov\\t%0.h[0], %1.h[0]
1190    fmov\\t%h0, %1
1191    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1192    ldr\\t%h0, %1
1193    str\\t%h1, %0
1194    ldrh\\t%w0, %1
1195    strh\\t%w1, %0
1196    mov\\t%w0, %w1"
1197   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1198                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1199    (set_attr "simd" "yes,*,yes,yes,yes,*,yes,*,*,*,*,*")
1200    (set_attr "fp16"   "*,yes,*,*,*,yes,*,*,*,*,*,*")]
1201 )
1202
1203 (define_insn "*movsf_aarch64"
1204   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1205         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1206   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1207     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1208   "@
1209    movi\\t%0.2s, #0
1210    fmov\\t%s0, %w1
1211    fmov\\t%w0, %s1
1212    fmov\\t%s0, %s1
1213    fmov\\t%s0, %1
1214    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1215    ldr\\t%s0, %1
1216    str\\t%s1, %0
1217    ldr\\t%w0, %1
1218    str\\t%w1, %0
1219    mov\\t%w0, %w1
1220    mov\\t%w0, %1"
1221   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1222                      f_loads,f_stores,load_4,store_4,mov_reg,\
1223                      fconsts")
1224    (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1225 )
1226
1227 (define_insn "*movdf_aarch64"
1228   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1229         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1230   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1231     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1232   "@
1233    movi\\t%d0, #0
1234    fmov\\t%d0, %x1
1235    fmov\\t%x0, %d1
1236    fmov\\t%d0, %d1
1237    fmov\\t%d0, %1
1238    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1239    ldr\\t%d0, %1
1240    str\\t%d1, %0
1241    ldr\\t%x0, %1
1242    str\\t%x1, %0
1243    mov\\t%x0, %x1
1244    mov\\t%x0, %1"
1245   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1246                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1247                      fconstd")
1248    (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1249 )
1250
1251 (define_split
1252   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1253         (match_operand:GPF_HF 1 "general_operand"))]
1254   "can_create_pseudo_p ()
1255    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1256    && !aarch64_float_const_representable_p (operands[1])
1257    &&  aarch64_float_const_rtx_p (operands[1])"
1258   [(const_int 0)]
1259   {
1260     unsigned HOST_WIDE_INT ival;
1261     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1262       FAIL;
1263
1264     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1265     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1266     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1267     DONE;
1268   }
1269 )
1270
1271 (define_insn "*movtf_aarch64"
1272   [(set (match_operand:TF 0
1273          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1274         (match_operand:TF 1
1275          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1276   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1277     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1278   "@
1279    mov\\t%0.16b, %1.16b
1280    #
1281    #
1282    #
1283    movi\\t%0.2d, #0
1284    fmov\\t%s0, wzr
1285    ldr\\t%q0, %1
1286    str\\t%q1, %0
1287    ldp\\t%0, %H0, %1
1288    stp\\t%1, %H1, %0
1289    stp\\txzr, xzr, %0"
1290   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1291                      f_loadd,f_stored,load_16,store_16,store_16")
1292    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1293    (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1294 )
1295
1296 (define_split
1297    [(set (match_operand:TF 0 "register_operand" "")
1298          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1299   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1300   [(const_int 0)]
1301   {
1302     aarch64_split_128bit_move (operands[0], operands[1]);
1303     DONE;
1304   }
1305 )
1306
1307 ;; 0 is dst
1308 ;; 1 is src
1309 ;; 2 is size of move in bytes
1310 ;; 3 is alignment
1311
1312 (define_expand "movmemdi"
1313   [(match_operand:BLK 0 "memory_operand")
1314    (match_operand:BLK 1 "memory_operand")
1315    (match_operand:DI 2 "immediate_operand")
1316    (match_operand:DI 3 "immediate_operand")]
1317    "!STRICT_ALIGNMENT"
1318 {
1319   if (aarch64_expand_movmem (operands))
1320     DONE;
1321   FAIL;
1322 }
1323 )
1324
1325 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1326 ;; fairly lax checking on the second memory operation.
1327 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1328   [(set (match_operand:SX 0 "register_operand" "=r,w")
1329         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1330    (set (match_operand:SX2 2 "register_operand" "=r,w")
1331         (match_operand:SX2 3 "memory_operand" "m,m"))]
1332    "rtx_equal_p (XEXP (operands[3], 0),
1333                  plus_constant (Pmode,
1334                                 XEXP (operands[1], 0),
1335                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1336   "@
1337    ldp\\t%w0, %w2, %1
1338    ldp\\t%s0, %s2, %1"
1339   [(set_attr "type" "load_8,neon_load1_2reg")
1340    (set_attr "fp" "*,yes")]
1341 )
1342
1343 ;; Storing different modes that can still be merged
1344 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1345   [(set (match_operand:DX 0 "register_operand" "=r,w")
1346         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1347    (set (match_operand:DX2 2 "register_operand" "=r,w")
1348         (match_operand:DX2 3 "memory_operand" "m,m"))]
1349    "rtx_equal_p (XEXP (operands[3], 0),
1350                  plus_constant (Pmode,
1351                                 XEXP (operands[1], 0),
1352                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1353   "@
1354    ldp\\t%x0, %x2, %1
1355    ldp\\t%d0, %d2, %1"
1356   [(set_attr "type" "load_16,neon_load1_2reg")
1357    (set_attr "fp" "*,yes")]
1358 )
1359
1360 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1361 ;; fairly lax checking on the second memory operation.
1362 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1363   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1364         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1365    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1366         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1367    "rtx_equal_p (XEXP (operands[2], 0),
1368                  plus_constant (Pmode,
1369                                 XEXP (operands[0], 0),
1370                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1371   "@
1372    stp\\t%w1, %w3, %0
1373    stp\\t%s1, %s3, %0"
1374   [(set_attr "type" "store_8,neon_store1_2reg")
1375    (set_attr "fp" "*,yes")]
1376 )
1377
1378 ;; Storing different modes that can still be merged
1379 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1380   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1381         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1382    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1383         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1384    "rtx_equal_p (XEXP (operands[2], 0),
1385                  plus_constant (Pmode,
1386                                 XEXP (operands[0], 0),
1387                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1388   "@
1389    stp\\t%x1, %x3, %0
1390    stp\\t%d1, %d3, %0"
1391   [(set_attr "type" "store_16,neon_store1_2reg")
1392    (set_attr "fp" "*,yes")]
1393 )
1394
1395 ;; Load pair with post-index writeback.  This is primarily used in function
1396 ;; epilogues.
1397 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1398   [(parallel
1399     [(set (match_operand:P 0 "register_operand" "=k")
1400           (plus:P (match_operand:P 1 "register_operand" "0")
1401                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1402      (set (match_operand:GPI 2 "register_operand" "=r")
1403           (mem:GPI (match_dup 1)))
1404      (set (match_operand:GPI 3 "register_operand" "=r")
1405           (mem:GPI (plus:P (match_dup 1)
1406                    (match_operand:P 5 "const_int_operand" "n"))))])]
1407   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1408   "ldp\\t%<w>2, %<w>3, [%1], %4"
1409   [(set_attr "type" "load_<ldpstp_sz>")]
1410 )
1411
1412 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1413   [(parallel
1414     [(set (match_operand:P 0 "register_operand" "=k")
1415           (plus:P (match_operand:P 1 "register_operand" "0")
1416                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1417      (set (match_operand:GPF 2 "register_operand" "=w")
1418           (mem:GPF (match_dup 1)))
1419      (set (match_operand:GPF 3 "register_operand" "=w")
1420           (mem:GPF (plus:P (match_dup 1)
1421                    (match_operand:P 5 "const_int_operand" "n"))))])]
1422   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1423   "ldp\\t%<w>2, %<w>3, [%1], %4"
1424   [(set_attr "type" "neon_load1_2reg")]
1425 )
1426
1427 ;; Store pair with pre-index writeback.  This is primarily used in function
1428 ;; prologues.
1429 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1430   [(parallel
1431     [(set (match_operand:P 0 "register_operand" "=&k")
1432           (plus:P (match_operand:P 1 "register_operand" "0")
1433                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1434      (set (mem:GPI (plus:P (match_dup 0)
1435                    (match_dup 4)))
1436           (match_operand:GPI 2 "register_operand" "r"))
1437      (set (mem:GPI (plus:P (match_dup 0)
1438                    (match_operand:P 5 "const_int_operand" "n")))
1439           (match_operand:GPI 3 "register_operand" "r"))])]
1440   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1441   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1442   [(set_attr "type" "store_<ldpstp_sz>")]
1443 )
1444
1445 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1446   [(parallel
1447     [(set (match_operand:P 0 "register_operand" "=&k")
1448           (plus:P (match_operand:P 1 "register_operand" "0")
1449                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1450      (set (mem:GPF (plus:P (match_dup 0)
1451                    (match_dup 4)))
1452           (match_operand:GPF 2 "register_operand" "w"))
1453      (set (mem:GPF (plus:P (match_dup 0)
1454                    (match_operand:P 5 "const_int_operand" "n")))
1455           (match_operand:GPF 3 "register_operand" "w"))])]
1456   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1457   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1458   [(set_attr "type" "neon_store1_2reg<q>")]
1459 )
1460
1461 ;; -------------------------------------------------------------------
1462 ;; Sign/Zero extension
1463 ;; -------------------------------------------------------------------
1464
1465 (define_expand "<optab>sidi2"
1466   [(set (match_operand:DI 0 "register_operand")
1467         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1468   ""
1469 )
1470
1471 (define_insn "*extendsidi2_aarch64"
1472   [(set (match_operand:DI 0 "register_operand" "=r,r")
1473         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1474   ""
1475   "@
1476    sxtw\t%0, %w1
1477    ldrsw\t%0, %1"
1478   [(set_attr "type" "extend,load_4")]
1479 )
1480
1481 (define_insn "*load_pair_extendsidi2_aarch64"
1482   [(set (match_operand:DI 0 "register_operand" "=r")
1483         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1484    (set (match_operand:DI 2 "register_operand" "=r")
1485         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1486   "rtx_equal_p (XEXP (operands[3], 0),
1487                 plus_constant (Pmode,
1488                                XEXP (operands[1], 0),
1489                                GET_MODE_SIZE (SImode)))"
1490   "ldpsw\\t%0, %2, %1"
1491   [(set_attr "type" "load_8")]
1492 )
1493
1494 (define_insn "*zero_extendsidi2_aarch64"
1495   [(set (match_operand:DI 0 "register_operand" "=r,r")
1496         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1497   ""
1498   "@
1499    uxtw\t%0, %w1
1500    ldr\t%w0, %1"
1501   [(set_attr "type" "extend,load_4")]
1502 )
1503
1504 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1505   [(set (match_operand:DI 0 "register_operand" "=r")
1506         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1507    (set (match_operand:DI 2 "register_operand" "=r")
1508         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1509   "rtx_equal_p (XEXP (operands[3], 0),
1510                 plus_constant (Pmode,
1511                                XEXP (operands[1], 0),
1512                                GET_MODE_SIZE (SImode)))"
1513   "ldp\\t%w0, %w2, %1"
1514   [(set_attr "type" "load_8")]
1515 )
1516
1517 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1518   [(set (match_operand:GPI 0 "register_operand")
1519         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1520   ""
1521 )
1522
1523 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1524   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1525         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1526   ""
1527   "@
1528    sxt<SHORT:size>\t%<GPI:w>0, %w1
1529    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1530   [(set_attr "type" "extend,load_4")]
1531 )
1532
1533 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1534   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1535         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1536   ""
1537   "@
1538    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1539    ldr<SHORT:size>\t%w0, %1
1540    ldr\t%<SHORT:size>0, %1"
1541   [(set_attr "type" "logic_imm,load_4,load_4")]
1542 )
1543
1544 (define_expand "<optab>qihi2"
1545   [(set (match_operand:HI 0 "register_operand")
1546         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1547   ""
1548 )
1549
1550 (define_insn "*extendqihi2_aarch64"
1551   [(set (match_operand:HI 0 "register_operand" "=r,r")
1552         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1553   ""
1554   "@
1555    sxtb\t%w0, %w1
1556    ldrsb\t%w0, %1"
1557   [(set_attr "type" "extend,load_4")]
1558 )
1559
1560 (define_insn "*zero_extendqihi2_aarch64"
1561   [(set (match_operand:HI 0 "register_operand" "=r,r")
1562         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1563   ""
1564   "@
1565    and\t%w0, %w1, 255
1566    ldrb\t%w0, %1"
1567   [(set_attr "type" "logic_imm,load_4")]
1568 )
1569
1570 ;; -------------------------------------------------------------------
1571 ;; Simple arithmetic
1572 ;; -------------------------------------------------------------------
1573
1574 (define_expand "add<mode>3"
1575   [(set
1576     (match_operand:GPI 0 "register_operand" "")
1577     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1578               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1579   ""
1580 {
1581   /* If operands[1] is a subreg extract the inner RTX.  */
1582   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1583
1584   /* If the constant is too large for a single instruction and isn't frame
1585      based, split off the immediate so it is available for CSE.  */
1586   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1587       && can_create_pseudo_p ()
1588       && (!REG_P (op1)
1589          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1590     operands[2] = force_reg (<MODE>mode, operands[2]);
1591   /* Expand polynomial additions now if the destination is the stack
1592      pointer, since we don't want to use that as a temporary.  */
1593   else if (operands[0] == stack_pointer_rtx
1594            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1595     {
1596       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1597                                 operands[2], NULL_RTX, NULL_RTX);
1598       DONE;
1599     }
1600 })
1601
1602 (define_insn "*add<mode>3_aarch64"
1603   [(set
1604     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1605     (plus:GPI
1606      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1607      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1608   ""
1609   "@
1610   add\\t%<w>0, %<w>1, %2
1611   add\\t%<w>0, %<w>1, %<w>2
1612   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1613   sub\\t%<w>0, %<w>1, #%n2
1614   #
1615   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1616   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1617   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1618    (set_attr "simd" "*,*,yes,*,*,*")]
1619 )
1620
1621 ;; zero_extend version of above
1622 (define_insn "*addsi3_aarch64_uxtw"
1623   [(set
1624     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1625     (zero_extend:DI
1626      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1627               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1628   ""
1629   "@
1630   add\\t%w0, %w1, %2
1631   add\\t%w0, %w1, %w2
1632   sub\\t%w0, %w1, #%n2
1633   #"
1634   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1635 )
1636
1637 ;; If there's a free register, and we can load the constant with a
1638 ;; single instruction, do so.  This has a chance to improve scheduling.
1639 (define_peephole2
1640   [(match_scratch:GPI 3 "r")
1641    (set (match_operand:GPI 0 "register_operand")
1642         (plus:GPI
1643           (match_operand:GPI 1 "register_operand")
1644           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1645   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1646   [(set (match_dup 3) (match_dup 2))
1647    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1648 )
1649
1650 (define_peephole2
1651   [(match_scratch:SI 3 "r")
1652    (set (match_operand:DI 0 "register_operand")
1653         (zero_extend:DI
1654           (plus:SI
1655             (match_operand:SI 1 "register_operand")
1656             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1657   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1658   [(set (match_dup 3) (match_dup 2))
1659    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1660 )
1661
1662 ;; After peephole2 has had a chance to run, split any remaining long
1663 ;; additions into two add immediates.
1664 (define_split
1665   [(set (match_operand:GPI 0 "register_operand")
1666         (plus:GPI
1667           (match_operand:GPI 1 "register_operand")
1668           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1669   "epilogue_completed"
1670   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1671    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1672   {
1673     HOST_WIDE_INT i = INTVAL (operands[2]);
1674     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1675     operands[3] = GEN_INT (i - s);
1676     operands[4] = GEN_INT (s);
1677   }
1678 )
1679
1680 ;; Match addition of polynomial offsets that require one temporary, for which
1681 ;; we can use the early-clobbered destination register.  This is a separate
1682 ;; pattern so that the early clobber doesn't affect register allocation
1683 ;; for other forms of addition.  However, we still need to provide an
1684 ;; all-register alternative, in case the offset goes out of range after
1685 ;; elimination.  For completeness we might as well provide all GPR-based
1686 ;; alternatives from the main pattern.
1687 ;;
1688 ;; We don't have a pattern for additions requiring two temporaries since at
1689 ;; present LRA doesn't allow new scratches to be added during elimination.
1690 ;; Such offsets should be rare anyway.
1691 ;;
1692 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1693 ;; here would go away.  We could just handle all polynomial constants in
1694 ;; this pattern.
1695 (define_insn_and_split "*add<mode>3_poly_1"
1696   [(set
1697     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1698     (plus:GPI
1699      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1700      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1701   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1702   "@
1703   add\\t%<w>0, %<w>1, %2
1704   add\\t%<w>0, %<w>1, %<w>2
1705   sub\\t%<w>0, %<w>1, #%n2
1706   #
1707   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1708   #"
1709   "&& epilogue_completed
1710    && !reg_overlap_mentioned_p (operands[0], operands[1])
1711    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1712   [(const_int 0)]
1713   {
1714     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1715                               operands[2], operands[0], NULL_RTX);
1716     DONE;
1717   }
1718   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1719   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1720 )
1721
1722 (define_split
1723   [(set (match_operand:DI 0 "register_operand")
1724         (zero_extend:DI
1725           (plus:SI
1726             (match_operand:SI 1 "register_operand")
1727             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1728   "epilogue_completed"
1729   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1730    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1731   {
1732     HOST_WIDE_INT i = INTVAL (operands[2]);
1733     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1734     operands[3] = GEN_INT (i - s);
1735     operands[4] = GEN_INT (s);
1736     operands[5] = gen_lowpart (SImode, operands[0]);
1737   }
1738 )
1739
1740 (define_expand "addv<mode>4"
1741   [(match_operand:GPI 0 "register_operand")
1742    (match_operand:GPI 1 "register_operand")
1743    (match_operand:GPI 2 "register_operand")
1744    (label_ref (match_operand 3 "" ""))]
1745   ""
1746 {
1747   emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1748   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1749
1750   DONE;
1751 })
1752
1753 (define_expand "uaddv<mode>4"
1754   [(match_operand:GPI 0 "register_operand")
1755    (match_operand:GPI 1 "register_operand")
1756    (match_operand:GPI 2 "register_operand")
1757    (label_ref (match_operand 3 "" ""))]
1758   ""
1759 {
1760   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1761   aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1762
1763   DONE;
1764 })
1765
1766 (define_expand "addti3"
1767   [(set (match_operand:TI 0 "register_operand" "")
1768         (plus:TI (match_operand:TI 1 "register_operand" "")
1769                  (match_operand:TI 2 "aarch64_reg_or_imm" "")))]
1770   ""
1771 {
1772   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1773
1774   aarch64_addti_scratch_regs (operands[1], operands[2],
1775                               &low_dest, &op1_low, &op2_low,
1776                               &high_dest, &op1_high, &op2_high);
1777
1778   if (op2_low == const0_rtx)
1779     {
1780       low_dest = op1_low;
1781       if (!aarch64_pluslong_operand (op2_high, DImode))
1782         op2_high = force_reg (DImode, op2_high);
1783       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1784     }
1785   else
1786     {
1787       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1788                                       force_reg (DImode, op2_low)));
1789       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1790                                      force_reg (DImode, op2_high)));
1791     }
1792
1793   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1794   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1795
1796   DONE;
1797 })
1798
1799 (define_expand "addvti4"
1800   [(match_operand:TI 0 "register_operand" "")
1801    (match_operand:TI 1 "register_operand" "")
1802    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1803    (label_ref (match_operand 3 "" ""))]
1804   ""
1805 {
1806   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1807
1808   aarch64_addti_scratch_regs (operands[1], operands[2],
1809                               &low_dest, &op1_low, &op2_low,
1810                               &high_dest, &op1_high, &op2_high);
1811
1812   if (op2_low == const0_rtx)
1813     {
1814       low_dest = op1_low;
1815       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1816                                       force_reg (DImode, op2_high)));
1817     }
1818   else
1819     {
1820       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1821                                       force_reg (DImode, op2_low)));
1822       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
1823                                       force_reg (DImode, op2_high)));
1824     }
1825
1826   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1827   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1828
1829   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1830   DONE;
1831 })
1832
1833 (define_expand "uaddvti4"
1834   [(match_operand:TI 0 "register_operand" "")
1835    (match_operand:TI 1 "register_operand" "")
1836    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1837    (label_ref (match_operand 3 "" ""))]
1838   ""
1839 {
1840   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1841
1842   aarch64_addti_scratch_regs (operands[1], operands[2],
1843                               &low_dest, &op1_low, &op2_low,
1844                               &high_dest, &op1_high, &op2_high);
1845
1846   if (op2_low == const0_rtx)
1847     {
1848       low_dest = op1_low;
1849       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
1850                                       force_reg (DImode, op2_high)));
1851     }
1852   else
1853     {
1854       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1855                                       force_reg (DImode, op2_low)));
1856       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
1857                                       force_reg (DImode, op2_high)));
1858     }
1859
1860   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1861   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1862
1863   aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1864   DONE;
1865  })
1866
1867 (define_insn "add<mode>3_compare0"
1868   [(set (reg:CC_NZ CC_REGNUM)
1869         (compare:CC_NZ
1870          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1871                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1872          (const_int 0)))
1873    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1874         (plus:GPI (match_dup 1) (match_dup 2)))]
1875   ""
1876   "@
1877   adds\\t%<w>0, %<w>1, %<w>2
1878   adds\\t%<w>0, %<w>1, %2
1879   subs\\t%<w>0, %<w>1, #%n2"
1880   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1881 )
1882
1883 ;; zero_extend version of above
1884 (define_insn "*addsi3_compare0_uxtw"
1885   [(set (reg:CC_NZ CC_REGNUM)
1886         (compare:CC_NZ
1887          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1888                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1889          (const_int 0)))
1890    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1891         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1892   ""
1893   "@
1894   adds\\t%w0, %w1, %w2
1895   adds\\t%w0, %w1, %2
1896   subs\\t%w0, %w1, #%n2"
1897   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1898 )
1899
1900 (define_insn "*add<mode>3_compareC_cconly_imm"
1901   [(set (reg:CC_C CC_REGNUM)
1902         (ne:CC_C
1903           (plus:<DWI>
1904             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1905             (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1906           (zero_extend:<DWI>
1907             (plus:GPI
1908               (match_dup 0)
1909               (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1910   "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1911                                  <MODE>mode, operands[1])"
1912   "@
1913   cmn\\t%<w>0, %1
1914   cmp\\t%<w>0, #%n1"
1915   [(set_attr "type" "alus_imm")]
1916 )
1917
1918 (define_insn "*add<mode>3_compareC_cconly"
1919   [(set (reg:CC_C CC_REGNUM)
1920         (ne:CC_C
1921           (plus:<DWI>
1922             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1923             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1924           (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1925   ""
1926   "cmn\\t%<w>0, %<w>1"
1927   [(set_attr "type" "alus_sreg")]
1928 )
1929
1930 (define_insn "*add<mode>3_compareC_imm"
1931   [(set (reg:CC_C CC_REGNUM)
1932         (ne:CC_C
1933           (plus:<DWI>
1934             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1935             (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1936           (zero_extend:<DWI>
1937             (plus:GPI
1938               (match_dup 1)
1939               (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1940    (set (match_operand:GPI 0 "register_operand" "=r,r")
1941         (plus:GPI (match_dup 1) (match_dup 2)))]
1942   "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1943                                  <MODE>mode, operands[2])"
1944   "@
1945   adds\\t%<w>0, %<w>1, %2
1946   subs\\t%<w>0, %<w>1, #%n2"
1947   [(set_attr "type" "alus_imm")]
1948 )
1949
1950 (define_insn "add<mode>3_compareC"
1951   [(set (reg:CC_C CC_REGNUM)
1952         (compare:CC_C
1953           (plus:<DWI>
1954             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1955             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1956           (zero_extend:<DWI>
1957             (plus:GPI (match_dup 1) (match_dup 2)))))
1958    (set (match_operand:GPI 0 "register_operand" "=r")
1959         (plus:GPI (match_dup 1) (match_dup 2)))]
1960   ""
1961   "adds\\t%<w>0, %<w>1, %<w>2"
1962   [(set_attr "type" "alus_sreg")]
1963 )
1964
1965 (define_insn "*add<mode>3_compareV_cconly_imm"
1966   [(set (reg:CC_V CC_REGNUM)
1967         (compare:CC_V
1968           (plus:<DWI>
1969             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1970             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
1971           (sign_extend:<DWI>
1972            (plus:GPI
1973             (match_dup 0)
1974             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
1975   "INTVAL (operands[1]) == INTVAL (operands[2])"
1976   "@
1977   cmn\\t%<w>0, %<w>1
1978   cmp\\t%<w>0, #%n1"
1979   [(set_attr "type" "alus_imm")]
1980 )
1981
1982 (define_insn "*add<mode>3_compareV_cconly"
1983   [(set (reg:CC_V CC_REGNUM)
1984         (compare:CC_V
1985           (plus:<DWI>
1986             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1987             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1988           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1989   ""
1990   "cmn\\t%<w>0, %<w>1"
1991   [(set_attr "type" "alus_sreg")]
1992 )
1993
1994 (define_insn "*add<mode>3_compareV_imm"
1995   [(set (reg:CC_V CC_REGNUM)
1996         (compare:CC_V
1997           (plus:<DWI>
1998             (sign_extend:<DWI>
1999               (match_operand:GPI 1 "register_operand" "r,r"))
2000             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2001           (sign_extend:<DWI>
2002             (plus:GPI (match_dup 1) (match_dup 2)))))
2003    (set (match_operand:GPI 0 "register_operand" "=r,r")
2004         (plus:GPI (match_dup 1) (match_dup 2)))]
2005    ""
2006    "@
2007    adds\\t%<w>0, %<w>1, %<w>2
2008    subs\\t%<w>0, %<w>1, #%n2"
2009   [(set_attr "type" "alus_imm,alus_imm")]
2010 )
2011
2012 (define_insn "add<mode>3_compareV"
2013   [(set (reg:CC_V CC_REGNUM)
2014         (compare:CC_V
2015           (plus:<DWI>
2016             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
2017             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2018           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2019    (set (match_operand:GPI 0 "register_operand" "=r")
2020         (plus:GPI (match_dup 1) (match_dup 2)))]
2021   ""
2022   "adds\\t%<w>0, %<w>1, %<w>2"
2023   [(set_attr "type" "alus_sreg")]
2024 )
2025
2026 (define_insn "*adds_shift_imm_<mode>"
2027   [(set (reg:CC_NZ CC_REGNUM)
2028         (compare:CC_NZ
2029          (plus:GPI (ASHIFT:GPI
2030                     (match_operand:GPI 1 "register_operand" "r")
2031                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2032                    (match_operand:GPI 3 "register_operand" "r"))
2033          (const_int 0)))
2034    (set (match_operand:GPI 0 "register_operand" "=r")
2035         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2036                   (match_dup 3)))]
2037   ""
2038   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2039   [(set_attr "type" "alus_shift_imm")]
2040 )
2041
2042 (define_insn "*subs_shift_imm_<mode>"
2043   [(set (reg:CC_NZ CC_REGNUM)
2044         (compare:CC_NZ
2045          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2046                     (ASHIFT:GPI
2047                      (match_operand:GPI 2 "register_operand" "r")
2048                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2049          (const_int 0)))
2050    (set (match_operand:GPI 0 "register_operand" "=r")
2051         (minus:GPI (match_dup 1)
2052                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2053   ""
2054   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2055   [(set_attr "type" "alus_shift_imm")]
2056 )
2057
2058 (define_insn "*adds_mul_imm_<mode>"
2059   [(set (reg:CC_NZ CC_REGNUM)
2060         (compare:CC_NZ
2061          (plus:GPI (mult:GPI
2062                     (match_operand:GPI 1 "register_operand" "r")
2063                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2064                    (match_operand:GPI 3 "register_operand" "r"))
2065          (const_int 0)))
2066    (set (match_operand:GPI 0 "register_operand" "=r")
2067         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2068                   (match_dup 3)))]
2069   ""
2070   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2071   [(set_attr "type" "alus_shift_imm")]
2072 )
2073
2074 (define_insn "*subs_mul_imm_<mode>"
2075   [(set (reg:CC_NZ CC_REGNUM)
2076         (compare:CC_NZ
2077          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2078                     (mult:GPI
2079                      (match_operand:GPI 2 "register_operand" "r")
2080                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2081          (const_int 0)))
2082    (set (match_operand:GPI 0 "register_operand" "=r")
2083         (minus:GPI (match_dup 1)
2084                    (mult:GPI (match_dup 2) (match_dup 3))))]
2085   ""
2086   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2087   [(set_attr "type" "alus_shift_imm")]
2088 )
2089
2090 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2091   [(set (reg:CC_NZ CC_REGNUM)
2092         (compare:CC_NZ
2093          (plus:GPI
2094           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2095           (match_operand:GPI 2 "register_operand" "r"))
2096         (const_int 0)))
2097    (set (match_operand:GPI 0 "register_operand" "=r")
2098         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2099   ""
2100   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2101   [(set_attr "type" "alus_ext")]
2102 )
2103
2104 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2105   [(set (reg:CC_NZ CC_REGNUM)
2106         (compare:CC_NZ
2107          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2108                     (ANY_EXTEND:GPI
2109                      (match_operand:ALLX 2 "register_operand" "r")))
2110         (const_int 0)))
2111    (set (match_operand:GPI 0 "register_operand" "=r")
2112         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2113   ""
2114   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2115   [(set_attr "type" "alus_ext")]
2116 )
2117
2118 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2119   [(set (reg:CC_NZ CC_REGNUM)
2120         (compare:CC_NZ
2121          (plus:GPI (ashift:GPI
2122                     (ANY_EXTEND:GPI
2123                      (match_operand:ALLX 1 "register_operand" "r"))
2124                     (match_operand 2 "aarch64_imm3" "Ui3"))
2125                    (match_operand:GPI 3 "register_operand" "r"))
2126          (const_int 0)))
2127    (set (match_operand:GPI 0 "register_operand" "=rk")
2128         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2129                               (match_dup 2))
2130                   (match_dup 3)))]
2131   ""
2132   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2133   [(set_attr "type" "alus_ext")]
2134 )
2135
2136 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2137   [(set (reg:CC_NZ CC_REGNUM)
2138         (compare:CC_NZ
2139          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2140                     (ashift:GPI
2141                      (ANY_EXTEND:GPI
2142                       (match_operand:ALLX 2 "register_operand" "r"))
2143                      (match_operand 3 "aarch64_imm3" "Ui3")))
2144          (const_int 0)))
2145    (set (match_operand:GPI 0 "register_operand" "=rk")
2146         (minus:GPI (match_dup 1)
2147                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2148                                (match_dup 3))))]
2149   ""
2150   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2151   [(set_attr "type" "alus_ext")]
2152 )
2153
2154 (define_insn "*adds_<optab><mode>_multp2"
2155   [(set (reg:CC_NZ CC_REGNUM)
2156         (compare:CC_NZ
2157          (plus:GPI (ANY_EXTRACT:GPI
2158                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2159                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2160                     (match_operand 3 "const_int_operand" "n")
2161                     (const_int 0))
2162                    (match_operand:GPI 4 "register_operand" "r"))
2163         (const_int 0)))
2164    (set (match_operand:GPI 0 "register_operand" "=r")
2165         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2166                                    (match_dup 3)
2167                                    (const_int 0))
2168                   (match_dup 4)))]
2169   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2170   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2171   [(set_attr "type" "alus_ext")]
2172 )
2173
2174 (define_insn "*subs_<optab><mode>_multp2"
2175   [(set (reg:CC_NZ CC_REGNUM)
2176         (compare:CC_NZ
2177          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
2178                     (ANY_EXTRACT:GPI
2179                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2180                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2181                      (match_operand 3 "const_int_operand" "n")
2182                      (const_int 0)))
2183         (const_int 0)))
2184    (set (match_operand:GPI 0 "register_operand" "=r")
2185         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2186                                   (mult:GPI (match_dup 1) (match_dup 2))
2187                                   (match_dup 3)
2188                                   (const_int 0))))]
2189   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2190   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2191   [(set_attr "type" "alus_ext")]
2192 )
2193
2194 (define_insn "*add<mode>3nr_compare0"
2195   [(set (reg:CC_NZ CC_REGNUM)
2196         (compare:CC_NZ
2197          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2198                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2199          (const_int 0)))]
2200   ""
2201   "@
2202   cmn\\t%<w>0, %<w>1
2203   cmn\\t%<w>0, %1
2204   cmp\\t%<w>0, #%n1"
2205   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2206 )
2207
2208 (define_insn "aarch64_sub<mode>_compare0"
2209   [(set (reg:CC_NZ CC_REGNUM)
2210         (compare:CC_NZ
2211          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2212                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2213          (const_int 0)))]
2214   ""
2215   "cmp\\t%<w>0, %<w>1"
2216   [(set_attr "type" "alus_sreg")]
2217 )
2218
2219 (define_insn "*compare_neg<mode>"
2220   [(set (reg:CC_Z CC_REGNUM)
2221         (compare:CC_Z
2222          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2223          (match_operand:GPI 1 "register_operand" "r")))]
2224   ""
2225   "cmn\\t%<w>1, %<w>0"
2226   [(set_attr "type" "alus_sreg")]
2227 )
2228
2229 (define_insn "*add_<shift>_<mode>"
2230   [(set (match_operand:GPI 0 "register_operand" "=r")
2231         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2232                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2233                   (match_operand:GPI 3 "register_operand" "r")))]
2234   ""
2235   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2236   [(set_attr "type" "alu_shift_imm")]
2237 )
2238
2239 ;; zero_extend version of above
2240 (define_insn "*add_<shift>_si_uxtw"
2241   [(set (match_operand:DI 0 "register_operand" "=r")
2242         (zero_extend:DI
2243          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2244                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2245                   (match_operand:SI 3 "register_operand" "r"))))]
2246   ""
2247   "add\\t%w0, %w3, %w1, <shift> %2"
2248   [(set_attr "type" "alu_shift_imm")]
2249 )
2250
2251 (define_insn "*add_mul_imm_<mode>"
2252   [(set (match_operand:GPI 0 "register_operand" "=r")
2253         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2254                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2255                   (match_operand:GPI 3 "register_operand" "r")))]
2256   ""
2257   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2258   [(set_attr "type" "alu_shift_imm")]
2259 )
2260
2261 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2262   [(set (match_operand:GPI 0 "register_operand" "=rk")
2263         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2264                   (match_operand:GPI 2 "register_operand" "r")))]
2265   ""
2266   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2267   [(set_attr "type" "alu_ext")]
2268 )
2269
2270 ;; zero_extend version of above
2271 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2272   [(set (match_operand:DI 0 "register_operand" "=rk")
2273         (zero_extend:DI
2274          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2275                   (match_operand:GPI 2 "register_operand" "r"))))]
2276   ""
2277   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2278   [(set_attr "type" "alu_ext")]
2279 )
2280
2281 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2282   [(set (match_operand:GPI 0 "register_operand" "=rk")
2283         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2284                                (match_operand:ALLX 1 "register_operand" "r"))
2285                               (match_operand 2 "aarch64_imm3" "Ui3"))
2286                   (match_operand:GPI 3 "register_operand" "r")))]
2287   ""
2288   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2289   [(set_attr "type" "alu_ext")]
2290 )
2291
2292 ;; zero_extend version of above
2293 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2294   [(set (match_operand:DI 0 "register_operand" "=rk")
2295         (zero_extend:DI
2296          (plus:SI (ashift:SI (ANY_EXTEND:SI
2297                               (match_operand:SHORT 1 "register_operand" "r"))
2298                              (match_operand 2 "aarch64_imm3" "Ui3"))
2299                   (match_operand:SI 3 "register_operand" "r"))))]
2300   ""
2301   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2302   [(set_attr "type" "alu_ext")]
2303 )
2304
2305 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2306   [(set (match_operand:GPI 0 "register_operand" "=rk")
2307         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2308                              (match_operand:ALLX 1 "register_operand" "r"))
2309                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2310                   (match_operand:GPI 3 "register_operand" "r")))]
2311   ""
2312   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2313   [(set_attr "type" "alu_ext")]
2314 )
2315
2316 ;; zero_extend version of above
2317 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2318   [(set (match_operand:DI 0 "register_operand" "=rk")
2319         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2320                              (match_operand:SHORT 1 "register_operand" "r"))
2321                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2322                   (match_operand:SI 3 "register_operand" "r"))))]
2323   ""
2324   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2325   [(set_attr "type" "alu_ext")]
2326 )
2327
2328 (define_insn "*add_<optab><mode>_multp2"
2329   [(set (match_operand:GPI 0 "register_operand" "=rk")
2330         (plus:GPI (ANY_EXTRACT:GPI
2331                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2332                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2333                    (match_operand 3 "const_int_operand" "n")
2334                    (const_int 0))
2335                   (match_operand:GPI 4 "register_operand" "r")))]
2336   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2337   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2338   [(set_attr "type" "alu_ext")]
2339 )
2340
2341 ;; zero_extend version of above
2342 (define_insn "*add_<optab>si_multp2_uxtw"
2343   [(set (match_operand:DI 0 "register_operand" "=rk")
2344         (zero_extend:DI
2345          (plus:SI (ANY_EXTRACT:SI
2346                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2347                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2348                    (match_operand 3 "const_int_operand" "n")
2349                    (const_int 0))
2350                   (match_operand:SI 4 "register_operand" "r"))))]
2351   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2352   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2353   [(set_attr "type" "alu_ext")]
2354 )
2355
2356 (define_expand "add<mode>3_carryin"
2357   [(set (match_operand:GPI 0 "register_operand")
2358         (plus:GPI
2359           (plus:GPI
2360             (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2361             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2362           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2363    ""
2364    ""
2365 )
2366
2367 ;; Note that add with carry with two zero inputs is matched by cset,
2368 ;; and that add with carry with one zero input is matched by cinc.
2369
2370 (define_insn "*add<mode>3_carryin"
2371   [(set (match_operand:GPI 0 "register_operand" "=r")
2372         (plus:GPI
2373           (plus:GPI
2374             (match_operand:GPI 3 "aarch64_carry_operation" "")
2375             (match_operand:GPI 1 "register_operand" "r"))
2376           (match_operand:GPI 2 "register_operand" "r")))]
2377    ""
2378    "adc\\t%<w>0, %<w>1, %<w>2"
2379   [(set_attr "type" "adc_reg")]
2380 )
2381
2382 ;; zero_extend version of above
2383 (define_insn "*addsi3_carryin_uxtw"
2384   [(set (match_operand:DI 0 "register_operand" "=r")
2385         (zero_extend:DI
2386           (plus:SI
2387             (plus:SI
2388               (match_operand:SI 3 "aarch64_carry_operation" "")
2389               (match_operand:SI 1 "register_operand" "r"))
2390             (match_operand:SI 2 "register_operand" "r"))))]
2391    ""
2392    "adc\\t%w0, %w1, %w2"
2393   [(set_attr "type" "adc_reg")]
2394 )
2395
2396 (define_expand "add<mode>3_carryinC"
2397   [(parallel
2398      [(set (match_dup 3)
2399            (compare:CC_C
2400              (plus:<DWI>
2401                (plus:<DWI>
2402                  (match_dup 4)
2403                  (zero_extend:<DWI>
2404                    (match_operand:GPI 1 "register_operand" "")))
2405                (zero_extend:<DWI>
2406                  (match_operand:GPI 2 "register_operand" "")))
2407            (zero_extend:<DWI>
2408              (plus:GPI
2409                (plus:GPI (match_dup 5) (match_dup 1))
2410                (match_dup 2)))))
2411       (set (match_operand:GPI 0 "register_operand")
2412            (plus:GPI
2413              (plus:GPI (match_dup 5) (match_dup 1))
2414              (match_dup 2)))])]
2415    ""
2416 {
2417   operands[3] = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2418   operands[4] = gen_rtx_NE (<DWI>mode, operands[3], const0_rtx);
2419   operands[5] = gen_rtx_NE (<MODE>mode, operands[3], const0_rtx);
2420 })
2421
2422 (define_insn "*add<mode>3_carryinC_zero"
2423   [(set (reg:CC_C CC_REGNUM)
2424         (compare:CC_C
2425           (plus:<DWI>
2426             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2427             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2428           (zero_extend:<DWI>
2429             (plus:GPI
2430               (match_operand:GPI 3 "aarch64_carry_operation" "")
2431               (match_dup 1)))))
2432    (set (match_operand:GPI 0 "register_operand")
2433         (plus:GPI (match_dup 3) (match_dup 1)))]
2434    ""
2435    "adcs\\t%<w>0, %<w>1, <w>zr"
2436   [(set_attr "type" "adc_reg")]
2437 )
2438
2439 (define_insn "*add<mode>3_carryinC"
2440   [(set (reg:CC_C CC_REGNUM)
2441         (compare:CC_C
2442           (plus:<DWI>
2443             (plus:<DWI>
2444               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2445               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2446             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2447           (zero_extend:<DWI>
2448             (plus:GPI
2449               (plus:GPI
2450                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2451                 (match_dup 1))
2452               (match_dup 2)))))
2453    (set (match_operand:GPI 0 "register_operand")
2454         (plus:GPI
2455           (plus:GPI (match_dup 4) (match_dup 1))
2456           (match_dup 2)))]
2457    ""
2458    "adcs\\t%<w>0, %<w>1, %<w>2"
2459   [(set_attr "type" "adc_reg")]
2460 )
2461
2462 (define_expand "add<mode>3_carryinV"
2463   [(parallel
2464      [(set (reg:CC_V CC_REGNUM)
2465            (compare:CC_V
2466              (plus:<DWI>
2467                (plus:<DWI>
2468                  (match_dup 3)
2469                  (sign_extend:<DWI>
2470                    (match_operand:GPI 1 "register_operand" "")))
2471                (sign_extend:<DWI>
2472                  (match_operand:GPI 2 "register_operand" "")))
2473            (sign_extend:<DWI>
2474              (plus:GPI
2475                (plus:GPI (match_dup 4) (match_dup 1))
2476                (match_dup 2)))))
2477       (set (match_operand:GPI 0 "register_operand")
2478            (plus:GPI
2479              (plus:GPI (match_dup 4) (match_dup 1))
2480              (match_dup 2)))])]
2481    ""
2482 {
2483   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2484   operands[3] = gen_rtx_NE (<DWI>mode, cc, const0_rtx);
2485   operands[4] = gen_rtx_NE (<MODE>mode, cc, const0_rtx);
2486 })
2487
2488 (define_insn "*add<mode>3_carryinV_zero"
2489   [(set (reg:CC_V CC_REGNUM)
2490         (compare:CC_V
2491           (plus:<DWI>
2492             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2493             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2494           (sign_extend:<DWI>
2495             (plus:GPI
2496               (match_operand:GPI 3 "aarch64_carry_operation" "")
2497               (match_dup 1)))))
2498    (set (match_operand:GPI 0 "register_operand")
2499         (plus:GPI (match_dup 3) (match_dup 1)))]
2500    ""
2501    "adcs\\t%<w>0, %<w>1, <w>zr"
2502   [(set_attr "type" "adc_reg")]
2503 )
2504
2505 (define_insn "*add<mode>3_carryinV"
2506   [(set (reg:CC_V CC_REGNUM)
2507         (compare:CC_V
2508           (plus:<DWI>
2509             (plus:<DWI>
2510               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2511               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2512             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2513           (sign_extend:<DWI>
2514             (plus:GPI
2515               (plus:GPI
2516                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2517                 (match_dup 1))
2518               (match_dup 2)))))
2519    (set (match_operand:GPI 0 "register_operand")
2520         (plus:GPI
2521           (plus:GPI (match_dup 4) (match_dup 1))
2522           (match_dup 2)))]
2523    ""
2524    "adcs\\t%<w>0, %<w>1, %<w>2"
2525   [(set_attr "type" "adc_reg")]
2526 )
2527
2528 (define_insn "*add_uxt<mode>_shift2"
2529   [(set (match_operand:GPI 0 "register_operand" "=rk")
2530         (plus:GPI (and:GPI
2531                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2532                                (match_operand 2 "aarch64_imm3" "Ui3"))
2533                    (match_operand 3 "const_int_operand" "n"))
2534                   (match_operand:GPI 4 "register_operand" "r")))]
2535   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2536   "*
2537   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2538                                            INTVAL (operands[3])));
2539   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2540   [(set_attr "type" "alu_ext")]
2541 )
2542
2543 ;; zero_extend version of above
2544 (define_insn "*add_uxtsi_shift2_uxtw"
2545   [(set (match_operand:DI 0 "register_operand" "=rk")
2546         (zero_extend:DI
2547          (plus:SI (and:SI
2548                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2549                               (match_operand 2 "aarch64_imm3" "Ui3"))
2550                    (match_operand 3 "const_int_operand" "n"))
2551                   (match_operand:SI 4 "register_operand" "r"))))]
2552   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2553   "*
2554   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2555                                            INTVAL (operands[3])));
2556   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2557   [(set_attr "type" "alu_ext")]
2558 )
2559
2560 (define_insn "*add_uxt<mode>_multp2"
2561   [(set (match_operand:GPI 0 "register_operand" "=rk")
2562         (plus:GPI (and:GPI
2563                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2564                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2565                    (match_operand 3 "const_int_operand" "n"))
2566                   (match_operand:GPI 4 "register_operand" "r")))]
2567   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2568   "*
2569   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2570                                            INTVAL (operands[3])));
2571   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2572   [(set_attr "type" "alu_ext")]
2573 )
2574
2575 ;; zero_extend version of above
2576 (define_insn "*add_uxtsi_multp2_uxtw"
2577   [(set (match_operand:DI 0 "register_operand" "=rk")
2578         (zero_extend:DI
2579          (plus:SI (and:SI
2580                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2581                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2582                    (match_operand 3 "const_int_operand" "n"))
2583                   (match_operand:SI 4 "register_operand" "r"))))]
2584   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2585   "*
2586   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2587                                            INTVAL (operands[3])));
2588   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2589   [(set_attr "type" "alu_ext")]
2590 )
2591
2592 (define_insn "subsi3"
2593   [(set (match_operand:SI 0 "register_operand" "=rk")
2594         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2595                   (match_operand:SI 2 "register_operand" "r")))]
2596   ""
2597   "sub\\t%w0, %w1, %w2"
2598   [(set_attr "type" "alu_sreg")]
2599 )
2600
2601 ;; zero_extend version of above
2602 (define_insn "*subsi3_uxtw"
2603   [(set (match_operand:DI 0 "register_operand" "=rk")
2604         (zero_extend:DI
2605          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2606                    (match_operand:SI 2 "register_operand" "r"))))]
2607   ""
2608   "sub\\t%w0, %w1, %w2"
2609   [(set_attr "type" "alu_sreg")]
2610 )
2611
2612 (define_insn "subdi3"
2613   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2614         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2615                   (match_operand:DI 2 "register_operand" "r,w")))]
2616   ""
2617   "@
2618    sub\\t%x0, %x1, %x2
2619    sub\\t%d0, %d1, %d2"
2620   [(set_attr "type" "alu_sreg, neon_sub")
2621    (set_attr "simd" "*,yes")]
2622 )
2623
2624 (define_expand "subv<mode>4"
2625   [(match_operand:GPI 0 "register_operand")
2626    (match_operand:GPI 1 "aarch64_reg_or_zero")
2627    (match_operand:GPI 2 "aarch64_reg_or_zero")
2628    (label_ref (match_operand 3 "" ""))]
2629   ""
2630 {
2631   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2632   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2633
2634   DONE;
2635 })
2636
2637 (define_expand "usubv<mode>4"
2638   [(match_operand:GPI 0 "register_operand")
2639    (match_operand:GPI 1 "aarch64_reg_or_zero")
2640    (match_operand:GPI 2 "aarch64_reg_or_zero")
2641    (label_ref (match_operand 3 "" ""))]
2642   ""
2643 {
2644   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2645   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2646
2647   DONE;
2648 })
2649
2650 (define_expand "subti3"
2651   [(set (match_operand:TI 0 "register_operand" "")
2652         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero" "")
2653                   (match_operand:TI 2 "register_operand" "")))]
2654   ""
2655 {
2656   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2657
2658   aarch64_subvti_scratch_regs (operands[1], operands[2],
2659                                &low_dest, &op1_low, &op2_low,
2660                                &high_dest, &op1_high, &op2_high);
2661
2662   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2663   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2664
2665   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2666   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2667   DONE;
2668 })
2669
2670 (define_expand "subvti4"
2671   [(match_operand:TI 0 "register_operand")
2672    (match_operand:TI 1 "aarch64_reg_or_zero")
2673    (match_operand:TI 2 "aarch64_reg_or_imm")
2674    (label_ref (match_operand 3 "" ""))]
2675   ""
2676 {
2677   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2678
2679   aarch64_subvti_scratch_regs (operands[1], operands[2],
2680                                &low_dest, &op1_low, &op2_low,
2681                                &high_dest, &op1_high, &op2_high);
2682   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2683                          high_dest, op1_high, op2_high);
2684
2685   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2686   DONE;
2687 })
2688
2689 (define_expand "usubvti4"
2690   [(match_operand:TI 0 "register_operand")
2691    (match_operand:TI 1 "aarch64_reg_or_zero")
2692    (match_operand:TI 2 "aarch64_reg_or_imm")
2693    (label_ref (match_operand 3 "" ""))]
2694   ""
2695 {
2696   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2697
2698   aarch64_subvti_scratch_regs (operands[1], operands[2],
2699                                     &low_dest, &op1_low, &op2_low,
2700                                &high_dest, &op1_high, &op2_high);
2701   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2702                          high_dest, op1_high, op2_high);
2703
2704   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2705   DONE;
2706 })
2707
2708 (define_insn "*sub<mode>3_compare0"
2709   [(set (reg:CC_NZ CC_REGNUM)
2710         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2711                                   (match_operand:GPI 2 "register_operand" "r"))
2712                        (const_int 0)))
2713    (set (match_operand:GPI 0 "register_operand" "=r")
2714         (minus:GPI (match_dup 1) (match_dup 2)))]
2715   ""
2716   "subs\\t%<w>0, %<w>1, %<w>2"
2717   [(set_attr "type" "alus_sreg")]
2718 )
2719
2720 ;; zero_extend version of above
2721 (define_insn "*subsi3_compare0_uxtw"
2722   [(set (reg:CC_NZ CC_REGNUM)
2723         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2724                                  (match_operand:SI 2 "register_operand" "r"))
2725                        (const_int 0)))
2726    (set (match_operand:DI 0 "register_operand" "=r")
2727         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2728   ""
2729   "subs\\t%w0, %w1, %w2"
2730   [(set_attr "type" "alus_sreg")]
2731 )
2732
2733 (define_insn "*sub<mode>3_compare1_imm"
2734   [(set (reg:CC CC_REGNUM)
2735         (compare:CC
2736           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ,rZ")
2737           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2738    (set (match_operand:GPI 0 "register_operand" "=r,r")
2739         (plus:GPI
2740           (match_dup 1)
2741           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
2742   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
2743   "@
2744   subs\\t%<w>0, %<w>1, #%n3
2745   adds\\t%<w>0, %<w>1, %3"
2746   [(set_attr "type" "alus_imm")]
2747 )
2748
2749 (define_insn "sub<mode>3_compare1"
2750   [(set (reg:CC CC_REGNUM)
2751         (compare:CC
2752           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2753           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2754    (set (match_operand:GPI 0 "register_operand" "=r")
2755         (minus:GPI (match_dup 1) (match_dup 2)))]
2756   ""
2757   "subs\\t%<w>0, %<w>1, %<w>2"
2758   [(set_attr "type" "alus_sreg")]
2759 )
2760
2761 (define_insn "sub<mode>3_compare1_imm"
2762   [(set (reg:CC CC_REGNUM)
2763         (compare:CC
2764           (match_operand:GPI 1 "register_operand" "r")
2765           (match_operand:GPI 3 "const_int_operand" "n")))
2766    (set (match_operand:GPI 0 "register_operand" "=r")
2767         (plus:GPI (match_dup 1)
2768                   (match_operand:GPI 2 "aarch64_sub_immediate" "J")))]
2769   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2770   "subs\\t%<w>0, %<w>1, #%n2"
2771   [(set_attr "type" "alus_sreg")]
2772 )
2773
2774 (define_peephole2
2775   [(set (match_operand:GPI 0 "register_operand")
2776         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
2777                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
2778    (set (reg:CC CC_REGNUM)
2779         (compare:CC
2780           (match_dup 1)
2781           (match_dup 2)))]
2782   "!reg_overlap_mentioned_p (operands[0], operands[1])
2783    && !reg_overlap_mentioned_p (operands[0], operands[2])"
2784   [(const_int 0)]
2785   {
2786     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2787                                          operands[2]));
2788     DONE;
2789   }
2790 )
2791
2792 ;; Same as the above peephole but with the compare and minus in
2793 ;; swapped order.  The restriction on overlap between operand 0
2794 ;; and operands 1 and 2 doesn't apply here.
2795 (define_peephole2
2796   [(set (reg:CC CC_REGNUM)
2797         (compare:CC
2798           (match_operand:GPI 1 "aarch64_reg_or_zero")
2799           (match_operand:GPI 2 "aarch64_reg_or_zero")))
2800    (set (match_operand:GPI 0 "register_operand")
2801         (minus:GPI (match_dup 1)
2802                    (match_dup 2)))]
2803   ""
2804   [(const_int 0)]
2805   {
2806     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2807                                          operands[2]));
2808     DONE;
2809   }
2810 )
2811
2812 (define_peephole2
2813   [(set (match_operand:GPI 0 "register_operand")
2814         (plus:GPI (match_operand:GPI 1 "register_operand")
2815                   (match_operand:GPI 2 "aarch64_sub_immediate")))
2816    (set (reg:CC CC_REGNUM)
2817         (compare:CC
2818           (match_dup 1)
2819           (match_operand:GPI 3 "const_int_operand")))]
2820   "!reg_overlap_mentioned_p (operands[0], operands[1])
2821    && INTVAL (operands[3]) == -INTVAL (operands[2])"
2822   [(const_int 0)]
2823   {
2824     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2825                                          operands[2], operands[3]));
2826     DONE;
2827   }
2828 )
2829
2830 ;; Same as the above peephole but with the compare and minus in
2831 ;; swapped order.  The restriction on overlap between operand 0
2832 ;; and operands 1 doesn't apply here.
2833 (define_peephole2
2834   [(set (reg:CC CC_REGNUM)
2835         (compare:CC
2836           (match_operand:GPI 1 "register_operand")
2837           (match_operand:GPI 3 "const_int_operand")))
2838    (set (match_operand:GPI 0 "register_operand")
2839         (plus:GPI (match_dup 1)
2840                   (match_operand:GPI 2 "aarch64_sub_immediate")))]
2841   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2842   [(const_int 0)]
2843   {
2844     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2845                                          operands[2], operands[3]));
2846     DONE;
2847   }
2848 )
2849
2850 (define_insn "*sub_<shift>_<mode>"
2851   [(set (match_operand:GPI 0 "register_operand" "=r")
2852         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2853                    (ASHIFT:GPI
2854                     (match_operand:GPI 1 "register_operand" "r")
2855                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2856   ""
2857   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2858   [(set_attr "type" "alu_shift_imm")]
2859 )
2860
2861 ;; zero_extend version of above
2862 (define_insn "*sub_<shift>_si_uxtw"
2863   [(set (match_operand:DI 0 "register_operand" "=r")
2864         (zero_extend:DI
2865          (minus:SI (match_operand:SI 3 "register_operand" "r")
2866                    (ASHIFT:SI
2867                     (match_operand:SI 1 "register_operand" "r")
2868                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2869   ""
2870   "sub\\t%w0, %w3, %w1, <shift> %2"
2871   [(set_attr "type" "alu_shift_imm")]
2872 )
2873
2874 (define_insn "*sub_mul_imm_<mode>"
2875   [(set (match_operand:GPI 0 "register_operand" "=r")
2876         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2877                    (mult:GPI
2878                     (match_operand:GPI 1 "register_operand" "r")
2879                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2880   ""
2881   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2882   [(set_attr "type" "alu_shift_imm")]
2883 )
2884
2885 ;; zero_extend version of above
2886 (define_insn "*sub_mul_imm_si_uxtw"
2887   [(set (match_operand:DI 0 "register_operand" "=r")
2888         (zero_extend:DI
2889          (minus:SI (match_operand:SI 3 "register_operand" "r")
2890                    (mult:SI
2891                     (match_operand:SI 1 "register_operand" "r")
2892                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2893   ""
2894   "sub\\t%w0, %w3, %w1, lsl %p2"
2895   [(set_attr "type" "alu_shift_imm")]
2896 )
2897
2898 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2899   [(set (match_operand:GPI 0 "register_operand" "=rk")
2900         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2901                    (ANY_EXTEND:GPI
2902                     (match_operand:ALLX 2 "register_operand" "r"))))]
2903   ""
2904   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2905   [(set_attr "type" "alu_ext")]
2906 )
2907
2908 ;; zero_extend version of above
2909 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2910   [(set (match_operand:DI 0 "register_operand" "=rk")
2911         (zero_extend:DI
2912          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2913                    (ANY_EXTEND:SI
2914                     (match_operand:SHORT 2 "register_operand" "r")))))]
2915   ""
2916   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2917   [(set_attr "type" "alu_ext")]
2918 )
2919
2920 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2921   [(set (match_operand:GPI 0 "register_operand" "=rk")
2922         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2923                    (ashift:GPI (ANY_EXTEND:GPI
2924                                 (match_operand:ALLX 2 "register_operand" "r"))
2925                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2926   ""
2927   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2928   [(set_attr "type" "alu_ext")]
2929 )
2930
2931 ;; zero_extend version of above
2932 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2933   [(set (match_operand:DI 0 "register_operand" "=rk")
2934         (zero_extend:DI
2935          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2936                    (ashift:SI (ANY_EXTEND:SI
2937                                (match_operand:SHORT 2 "register_operand" "r"))
2938                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2939   ""
2940   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2941   [(set_attr "type" "alu_ext")]
2942 )
2943
2944 (define_insn "*sub_<optab><mode>_multp2"
2945   [(set (match_operand:GPI 0 "register_operand" "=rk")
2946         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2947                    (ANY_EXTRACT:GPI
2948                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2949                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2950                     (match_operand 3 "const_int_operand" "n")
2951                     (const_int 0))))]
2952   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2953   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2954   [(set_attr "type" "alu_ext")]
2955 )
2956
2957 ;; zero_extend version of above
2958 (define_insn "*sub_<optab>si_multp2_uxtw"
2959   [(set (match_operand:DI 0 "register_operand" "=rk")
2960         (zero_extend:DI
2961          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2962                    (ANY_EXTRACT:SI
2963                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2964                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2965                     (match_operand 3 "const_int_operand" "n")
2966                     (const_int 0)))))]
2967   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2968   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2969   [(set_attr "type" "alu_ext")]
2970 )
2971
2972 ;; The hardware description is op1 + ~op2 + C.
2973 ;;                           = op1 + (-op2 + 1) + (1 - !C)
2974 ;;                           = op1 - op2 - 1 + 1 - !C
2975 ;;                           = op1 - op2 - !C.
2976 ;; We describe the latter.
2977
2978 (define_insn "*sub<mode>3_carryin0"
2979   [(set (match_operand:GPI 0 "register_operand" "=r")
2980         (minus:GPI
2981           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2982           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
2983    ""
2984    "sbc\\t%<w>0, %<w>1, <w>zr"
2985   [(set_attr "type" "adc_reg")]
2986 )
2987
2988 ;; zero_extend version of the above
2989 (define_insn "*subsi3_carryin_uxtw"
2990   [(set (match_operand:DI 0 "register_operand" "=r")
2991         (zero_extend:DI
2992           (minus:SI
2993             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2994             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
2995    ""
2996    "sbc\\t%w0, %w1, wzr"
2997   [(set_attr "type" "adc_reg")]
2998 )
2999
3000 (define_expand "sub<mode>3_carryin"
3001   [(set (match_operand:GPI 0 "register_operand")
3002         (minus:GPI
3003           (minus:GPI
3004             (match_operand:GPI 1 "aarch64_reg_or_zero")
3005             (match_operand:GPI 2 "register_operand"))
3006           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3007    ""
3008    ""
3009 )
3010
3011 (define_insn "*sub<mode>3_carryin"
3012   [(set (match_operand:GPI 0 "register_operand" "=r")
3013         (minus:GPI
3014           (minus:GPI
3015             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3016             (match_operand:GPI 2 "register_operand" "r"))
3017           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3018
3019    ""
3020    "sbc\\t%<w>0, %<w>1, %<w>2"
3021   [(set_attr "type" "adc_reg")]
3022 )
3023
3024 ;; zero_extend version of the above
3025 (define_insn "*subsi3_carryin_uxtw"
3026   [(set (match_operand:DI 0 "register_operand" "=r")
3027         (zero_extend:DI
3028           (minus:SI
3029             (minus:SI
3030               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3031               (match_operand:SI 2 "register_operand" "r"))
3032             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3033
3034    ""
3035    "sbc\\t%w0, %w1, %w2"
3036   [(set_attr "type" "adc_reg")]
3037 )
3038
3039 (define_insn "*sub<mode>3_carryin_alt"
3040   [(set (match_operand:GPI 0 "register_operand" "=r")
3041         (minus:GPI
3042           (minus:GPI
3043             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3044             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3045           (match_operand:GPI 2 "register_operand" "r")))]
3046    ""
3047    "sbc\\t%<w>0, %<w>1, %<w>2"
3048   [(set_attr "type" "adc_reg")]
3049 )
3050
3051 ;; zero_extend version of the above
3052 (define_insn "*subsi3_carryin_alt_uxtw"
3053   [(set (match_operand:DI 0 "register_operand" "=r")
3054         (zero_extend:DI
3055           (minus:SI
3056             (minus:SI
3057               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3058               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3059             (match_operand:SI 2 "register_operand" "r"))))]
3060    ""
3061    "sbc\\t%w0, %w1, %w2"
3062   [(set_attr "type" "adc_reg")]
3063 )
3064
3065 (define_expand "sub<mode>3_carryinCV"
3066   [(parallel
3067      [(set (reg:CC CC_REGNUM)
3068            (compare:CC
3069              (sign_extend:<DWI>
3070                (match_operand:GPI 1 "aarch64_reg_or_zero" ""))
3071              (plus:<DWI>
3072                (sign_extend:<DWI>
3073                  (match_operand:GPI 2 "register_operand" ""))
3074                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3075       (set (match_operand:GPI 0 "register_operand" "")
3076            (minus:GPI
3077              (minus:GPI (match_dup 1) (match_dup 2))
3078              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3079    ""
3080 )
3081
3082 (define_insn "*sub<mode>3_carryinCV_z1_z2"
3083   [(set (reg:CC CC_REGNUM)
3084         (compare:CC
3085           (const_int 0)
3086           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3087    (set (match_operand:GPI 0 "register_operand" "=r")
3088         (neg:GPI (match_operand:GPI 1 "aarch64_borrow_operation" "")))]
3089    ""
3090    "sbcs\\t%<w>0, <w>zr, <w>zr"
3091   [(set_attr "type" "adc_reg")]
3092 )
3093
3094 (define_insn "*sub<mode>3_carryinCV_z1"
3095   [(set (reg:CC CC_REGNUM)
3096         (compare:CC
3097           (const_int 0)
3098           (plus:<DWI>
3099             (sign_extend:<DWI>
3100               (match_operand:GPI 1 "register_operand" "r"))
3101             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3102    (set (match_operand:GPI 0 "register_operand" "=r")
3103         (minus:GPI
3104           (neg:GPI (match_dup 1))
3105           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3106    ""
3107    "sbcs\\t%<w>0, <w>zr, %<w>1"
3108   [(set_attr "type" "adc_reg")]
3109 )
3110
3111 (define_insn "*sub<mode>3_carryinCV_z2"
3112   [(set (reg:CC CC_REGNUM)
3113         (compare:CC
3114           (sign_extend:<DWI>
3115             (match_operand:GPI 1 "register_operand" "r"))
3116           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3117    (set (match_operand:GPI 0 "register_operand" "=r")
3118         (minus:GPI
3119           (match_dup 1)
3120           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3121    ""
3122    "sbcs\\t%<w>0, %<w>1, <w>zr"
3123   [(set_attr "type" "adc_reg")]
3124 )
3125
3126 (define_insn "*sub<mode>3_carryinCV"
3127   [(set (reg:CC CC_REGNUM)
3128         (compare:CC
3129           (sign_extend:<DWI>
3130             (match_operand:GPI 1 "register_operand" "r"))
3131           (plus:<DWI>
3132             (sign_extend:<DWI>
3133               (match_operand:GPI 2 "register_operand" "r"))
3134             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3135    (set (match_operand:GPI 0 "register_operand" "=r")
3136         (minus:GPI
3137           (minus:GPI (match_dup 1) (match_dup 2))
3138           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3139    ""
3140    "sbcs\\t%<w>0, %<w>1, %<w>2"
3141   [(set_attr "type" "adc_reg")]
3142 )
3143
3144 (define_insn "*sub_uxt<mode>_shift2"
3145   [(set (match_operand:GPI 0 "register_operand" "=rk")
3146         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3147                    (and:GPI
3148                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3149                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3150                     (match_operand 3 "const_int_operand" "n"))))]
3151   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3152   "*
3153   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3154                                            INTVAL (operands[3])));
3155   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3156   [(set_attr "type" "alu_ext")]
3157 )
3158
3159 ;; zero_extend version of above
3160 (define_insn "*sub_uxtsi_shift2_uxtw"
3161   [(set (match_operand:DI 0 "register_operand" "=rk")
3162         (zero_extend:DI
3163          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3164                    (and:SI
3165                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3166                                (match_operand 2 "aarch64_imm3" "Ui3"))
3167                     (match_operand 3 "const_int_operand" "n")))))]
3168   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3169   "*
3170   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3171                                            INTVAL (operands[3])));
3172   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3173   [(set_attr "type" "alu_ext")]
3174 )
3175
3176 (define_insn "*sub_uxt<mode>_multp2"
3177   [(set (match_operand:GPI 0 "register_operand" "=rk")
3178         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3179                    (and:GPI
3180                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3181                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3182                     (match_operand 3 "const_int_operand" "n"))))]
3183   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3184   "*
3185   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3186                                            INTVAL (operands[3])));
3187   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3188   [(set_attr "type" "alu_ext")]
3189 )
3190
3191 ;; zero_extend version of above
3192 (define_insn "*sub_uxtsi_multp2_uxtw"
3193   [(set (match_operand:DI 0 "register_operand" "=rk")
3194         (zero_extend:DI
3195          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3196                    (and:SI
3197                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3198                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3199                     (match_operand 3 "const_int_operand" "n")))))]
3200   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3201   "*
3202   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3203                                            INTVAL (operands[3])));
3204   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3205   [(set_attr "type" "alu_ext")]
3206 )
3207
3208 (define_expand "abs<mode>2"
3209   [(match_operand:GPI 0 "register_operand" "")
3210    (match_operand:GPI 1 "register_operand" "")]
3211   ""
3212   {
3213     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3214     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3215     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3216     DONE;
3217   }
3218 )
3219
3220 (define_insn "neg<mode>2"
3221   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3222         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3223   ""
3224   "@
3225    neg\\t%<w>0, %<w>1
3226    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3227   [(set_attr "type" "alu_sreg, neon_neg<q>")
3228    (set_attr "simd" "*,yes")]
3229 )
3230
3231 ;; zero_extend version of above
3232 (define_insn "*negsi2_uxtw"
3233   [(set (match_operand:DI 0 "register_operand" "=r")
3234         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3235   ""
3236   "neg\\t%w0, %w1"
3237   [(set_attr "type" "alu_sreg")]
3238 )
3239
3240 (define_insn "*ngc<mode>"
3241   [(set (match_operand:GPI 0 "register_operand" "=r")
3242         (minus:GPI
3243           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3244           (match_operand:GPI 1 "register_operand" "r")))]
3245   ""
3246   "ngc\\t%<w>0, %<w>1"
3247   [(set_attr "type" "adc_reg")]
3248 )
3249
3250 (define_insn "*ngcsi_uxtw"
3251   [(set (match_operand:DI 0 "register_operand" "=r")
3252         (zero_extend:DI
3253           (minus:SI
3254             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3255             (match_operand:SI 1 "register_operand" "r"))))]
3256   ""
3257   "ngc\\t%w0, %w1"
3258   [(set_attr "type" "adc_reg")]
3259 )
3260
3261 (define_insn "neg<mode>2_compare0"
3262   [(set (reg:CC_NZ CC_REGNUM)
3263         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3264                        (const_int 0)))
3265    (set (match_operand:GPI 0 "register_operand" "=r")
3266         (neg:GPI (match_dup 1)))]
3267   ""
3268   "negs\\t%<w>0, %<w>1"
3269   [(set_attr "type" "alus_sreg")]
3270 )
3271
3272 ;; zero_extend version of above
3273 (define_insn "*negsi2_compare0_uxtw"
3274   [(set (reg:CC_NZ CC_REGNUM)
3275         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3276                        (const_int 0)))
3277    (set (match_operand:DI 0 "register_operand" "=r")
3278         (zero_extend:DI (neg:SI (match_dup 1))))]
3279   ""
3280   "negs\\t%w0, %w1"
3281   [(set_attr "type" "alus_sreg")]
3282 )
3283
3284 (define_insn "*neg_<shift><mode>3_compare0"
3285   [(set (reg:CC_NZ CC_REGNUM)
3286         (compare:CC_NZ
3287          (neg:GPI (ASHIFT:GPI
3288                    (match_operand:GPI 1 "register_operand" "r")
3289                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3290          (const_int 0)))
3291    (set (match_operand:GPI 0 "register_operand" "=r")
3292         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3293   ""
3294   "negs\\t%<w>0, %<w>1, <shift> %2"
3295   [(set_attr "type" "alus_shift_imm")]
3296 )
3297
3298 (define_insn "*neg_<shift>_<mode>2"
3299   [(set (match_operand:GPI 0 "register_operand" "=r")
3300         (neg:GPI (ASHIFT:GPI
3301                   (match_operand:GPI 1 "register_operand" "r")
3302                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3303   ""
3304   "neg\\t%<w>0, %<w>1, <shift> %2"
3305   [(set_attr "type" "alu_shift_imm")]
3306 )
3307
3308 ;; zero_extend version of above
3309 (define_insn "*neg_<shift>_si2_uxtw"
3310   [(set (match_operand:DI 0 "register_operand" "=r")
3311         (zero_extend:DI
3312          (neg:SI (ASHIFT:SI
3313                   (match_operand:SI 1 "register_operand" "r")
3314                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3315   ""
3316   "neg\\t%w0, %w1, <shift> %2"
3317   [(set_attr "type" "alu_shift_imm")]
3318 )
3319
3320 (define_insn "*neg_mul_imm_<mode>2"
3321   [(set (match_operand:GPI 0 "register_operand" "=r")
3322         (neg:GPI (mult:GPI
3323                   (match_operand:GPI 1 "register_operand" "r")
3324                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3325   ""
3326   "neg\\t%<w>0, %<w>1, lsl %p2"
3327   [(set_attr "type" "alu_shift_imm")]
3328 )
3329
3330 ;; zero_extend version of above
3331 (define_insn "*neg_mul_imm_si2_uxtw"
3332   [(set (match_operand:DI 0 "register_operand" "=r")
3333         (zero_extend:DI
3334          (neg:SI (mult:SI
3335                   (match_operand:SI 1 "register_operand" "r")
3336                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3337   ""
3338   "neg\\t%w0, %w1, lsl %p2"
3339   [(set_attr "type" "alu_shift_imm")]
3340 )
3341
3342 (define_insn "mul<mode>3"
3343   [(set (match_operand:GPI 0 "register_operand" "=r")
3344         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3345                   (match_operand:GPI 2 "register_operand" "r")))]
3346   ""
3347   "mul\\t%<w>0, %<w>1, %<w>2"
3348   [(set_attr "type" "mul")]
3349 )
3350
3351 ;; zero_extend version of above
3352 (define_insn "*mulsi3_uxtw"
3353   [(set (match_operand:DI 0 "register_operand" "=r")
3354         (zero_extend:DI
3355          (mult:SI (match_operand:SI 1 "register_operand" "r")
3356                   (match_operand:SI 2 "register_operand" "r"))))]
3357   ""
3358   "mul\\t%w0, %w1, %w2"
3359   [(set_attr "type" "mul")]
3360 )
3361
3362 (define_insn "madd<mode>"
3363   [(set (match_operand:GPI 0 "register_operand" "=r")
3364         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3365                             (match_operand:GPI 2 "register_operand" "r"))
3366                   (match_operand:GPI 3 "register_operand" "r")))]
3367   ""
3368   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3369   [(set_attr "type" "mla")]
3370 )
3371
3372 ;; zero_extend version of above
3373 (define_insn "*maddsi_uxtw"
3374   [(set (match_operand:DI 0 "register_operand" "=r")
3375         (zero_extend:DI
3376          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3377                            (match_operand:SI 2 "register_operand" "r"))
3378                   (match_operand:SI 3 "register_operand" "r"))))]
3379   ""
3380   "madd\\t%w0, %w1, %w2, %w3"
3381   [(set_attr "type" "mla")]
3382 )
3383
3384 (define_insn "*msub<mode>"
3385   [(set (match_operand:GPI 0 "register_operand" "=r")
3386         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3387                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3388                              (match_operand:GPI 2 "register_operand" "r"))))]
3389
3390   ""
3391   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3392   [(set_attr "type" "mla")]
3393 )
3394
3395 ;; zero_extend version of above
3396 (define_insn "*msubsi_uxtw"
3397   [(set (match_operand:DI 0 "register_operand" "=r")
3398         (zero_extend:DI
3399          (minus:SI (match_operand:SI 3 "register_operand" "r")
3400                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3401                             (match_operand:SI 2 "register_operand" "r")))))]
3402
3403   ""
3404   "msub\\t%w0, %w1, %w2, %w3"
3405   [(set_attr "type" "mla")]
3406 )
3407
3408 (define_insn "*mul<mode>_neg"
3409   [(set (match_operand:GPI 0 "register_operand" "=r")
3410         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3411                   (match_operand:GPI 2 "register_operand" "r")))]
3412
3413   ""
3414   "mneg\\t%<w>0, %<w>1, %<w>2"
3415   [(set_attr "type" "mul")]
3416 )
3417
3418 ;; zero_extend version of above
3419 (define_insn "*mulsi_neg_uxtw"
3420   [(set (match_operand:DI 0 "register_operand" "=r")
3421         (zero_extend:DI
3422          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3423                   (match_operand:SI 2 "register_operand" "r"))))]
3424
3425   ""
3426   "mneg\\t%w0, %w1, %w2"
3427   [(set_attr "type" "mul")]
3428 )
3429
3430 (define_insn "<su_optab>mulsidi3"
3431   [(set (match_operand:DI 0 "register_operand" "=r")
3432         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3433                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3434   ""
3435   "<su>mull\\t%0, %w1, %w2"
3436   [(set_attr "type" "<su>mull")]
3437 )
3438
3439 (define_insn "<su_optab>maddsidi4"
3440   [(set (match_operand:DI 0 "register_operand" "=r")
3441         (plus:DI (mult:DI
3442                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3443                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3444                  (match_operand:DI 3 "register_operand" "r")))]
3445   ""
3446   "<su>maddl\\t%0, %w1, %w2, %3"
3447   [(set_attr "type" "<su>mlal")]
3448 )
3449
3450 (define_insn "<su_optab>msubsidi4"
3451   [(set (match_operand:DI 0 "register_operand" "=r")
3452         (minus:DI
3453          (match_operand:DI 3 "register_operand" "r")
3454          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3455                   (ANY_EXTEND:DI
3456                    (match_operand:SI 2 "register_operand" "r")))))]
3457   ""
3458   "<su>msubl\\t%0, %w1, %w2, %3"
3459   [(set_attr "type" "<su>mlal")]
3460 )
3461
3462 (define_insn "*<su_optab>mulsidi_neg"
3463   [(set (match_operand:DI 0 "register_operand" "=r")
3464         (mult:DI (neg:DI
3465                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3466                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3467   ""
3468   "<su>mnegl\\t%0, %w1, %w2"
3469   [(set_attr "type" "<su>mull")]
3470 )
3471
3472 (define_expand "<su_optab>mulditi3"
3473   [(set (match_operand:TI 0 "register_operand")
3474         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3475                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3476   ""
3477 {
3478   rtx low = gen_reg_rtx (DImode);
3479   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3480
3481   rtx high = gen_reg_rtx (DImode);
3482   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3483
3484   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3485   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3486   DONE;
3487 })
3488
3489 ;; The default expansion of multi3 using umuldi3_highpart will perform
3490 ;; the additions in an order that fails to combine into two madd insns.
3491 (define_expand "multi3"
3492   [(set (match_operand:TI 0 "register_operand")
3493         (mult:TI (match_operand:TI 1 "register_operand")
3494                  (match_operand:TI 2 "register_operand")))]
3495   ""
3496 {
3497   rtx l0 = gen_reg_rtx (DImode);
3498   rtx l1 = gen_lowpart (DImode, operands[1]);
3499   rtx l2 = gen_lowpart (DImode, operands[2]);
3500   rtx h0 = gen_reg_rtx (DImode);
3501   rtx h1 = gen_highpart (DImode, operands[1]);
3502   rtx h2 = gen_highpart (DImode, operands[2]);
3503
3504   emit_insn (gen_muldi3 (l0, l1, l2));
3505   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3506   emit_insn (gen_madddi (h0, h1, l2, h0));
3507   emit_insn (gen_madddi (h0, l1, h2, h0));
3508
3509   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3510   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3511   DONE;
3512 })
3513
3514 (define_insn "<su>muldi3_highpart"
3515   [(set (match_operand:DI 0 "register_operand" "=r")
3516         (truncate:DI
3517          (lshiftrt:TI
3518           (mult:TI
3519            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3520            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3521           (const_int 64))))]
3522   ""
3523   "<su>mulh\\t%0, %1, %2"
3524   [(set_attr "type" "<su>mull")]
3525 )
3526
3527 (define_insn "<su_optab>div<mode>3"
3528   [(set (match_operand:GPI 0 "register_operand" "=r")
3529         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3530                      (match_operand:GPI 2 "register_operand" "r")))]
3531   ""
3532   "<su>div\\t%<w>0, %<w>1, %<w>2"
3533   [(set_attr "type" "<su>div")]
3534 )
3535
3536 ;; zero_extend version of above
3537 (define_insn "*<su_optab>divsi3_uxtw"
3538   [(set (match_operand:DI 0 "register_operand" "=r")
3539         (zero_extend:DI
3540          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3541                      (match_operand:SI 2 "register_operand" "r"))))]
3542   ""
3543   "<su>div\\t%w0, %w1, %w2"
3544   [(set_attr "type" "<su>div")]
3545 )
3546
3547 ;; -------------------------------------------------------------------
3548 ;; Comparison insns
3549 ;; -------------------------------------------------------------------
3550
3551 (define_insn "cmp<mode>"
3552   [(set (reg:CC CC_REGNUM)
3553         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3554                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3555   ""
3556   "@
3557    cmp\\t%<w>0, %<w>1
3558    cmp\\t%<w>0, %1
3559    cmn\\t%<w>0, #%n1"
3560   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3561 )
3562
3563 (define_insn "fcmp<mode>"
3564   [(set (reg:CCFP CC_REGNUM)
3565         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3566                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3567    "TARGET_FLOAT"
3568    "@
3569     fcmp\\t%<s>0, #0.0
3570     fcmp\\t%<s>0, %<s>1"
3571   [(set_attr "type" "fcmp<s>")]
3572 )
3573
3574 (define_insn "fcmpe<mode>"
3575   [(set (reg:CCFPE CC_REGNUM)
3576         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3577                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3578    "TARGET_FLOAT"
3579    "@
3580     fcmpe\\t%<s>0, #0.0
3581     fcmpe\\t%<s>0, %<s>1"
3582   [(set_attr "type" "fcmp<s>")]
3583 )
3584
3585 (define_insn "*cmp_swp_<shift>_reg<mode>"
3586   [(set (reg:CC_SWP CC_REGNUM)
3587         (compare:CC_SWP (ASHIFT:GPI
3588                          (match_operand:GPI 0 "register_operand" "r")
3589                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3590                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3591   ""
3592   "cmp\\t%<w>2, %<w>0, <shift> %1"
3593   [(set_attr "type" "alus_shift_imm")]
3594 )
3595
3596 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3597   [(set (reg:CC_SWP CC_REGNUM)
3598         (compare:CC_SWP (ANY_EXTEND:GPI
3599                          (match_operand:ALLX 0 "register_operand" "r"))
3600                         (match_operand:GPI 1 "register_operand" "r")))]
3601   ""
3602   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3603   [(set_attr "type" "alus_ext")]
3604 )
3605
3606 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3607   [(set (reg:CC_SWP CC_REGNUM)
3608         (compare:CC_SWP (ashift:GPI
3609                          (ANY_EXTEND:GPI
3610                           (match_operand:ALLX 0 "register_operand" "r"))
3611                          (match_operand 1 "aarch64_imm3" "Ui3"))
3612         (match_operand:GPI 2 "register_operand" "r")))]
3613   ""
3614   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3615   [(set_attr "type" "alus_ext")]
3616 )
3617
3618 ;; -------------------------------------------------------------------
3619 ;; Store-flag and conditional select insns
3620 ;; -------------------------------------------------------------------
3621
3622 (define_expand "cstore<mode>4"
3623   [(set (match_operand:SI 0 "register_operand" "")
3624         (match_operator:SI 1 "aarch64_comparison_operator"
3625          [(match_operand:GPI 2 "register_operand" "")
3626           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3627   ""
3628   "
3629   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3630                                       operands[3]);
3631   operands[3] = const0_rtx;
3632   "
3633 )
3634
3635 (define_expand "cstorecc4"
3636   [(set (match_operand:SI 0 "register_operand")
3637        (match_operator 1 "aarch64_comparison_operator_mode"
3638         [(match_operand 2 "cc_register")
3639          (match_operand 3 "const0_operand")]))]
3640   ""
3641 "{
3642   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3643   DONE;
3644 }")
3645
3646
3647 (define_expand "cstore<mode>4"
3648   [(set (match_operand:SI 0 "register_operand" "")
3649         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3650          [(match_operand:GPF 2 "register_operand" "")
3651           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3652   ""
3653   "
3654   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3655                                       operands[3]);
3656   operands[3] = const0_rtx;
3657   "
3658 )
3659
3660 (define_insn "aarch64_cstore<mode>"
3661   [(set (match_operand:ALLI 0 "register_operand" "=r")
3662         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3663          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3664   ""
3665   "cset\\t%<w>0, %m1"
3666   [(set_attr "type" "csel")]
3667 )
3668
3669 ;; For a 24-bit immediate CST we can optimize the compare for equality
3670 ;; and branch sequence from:
3671 ;;      mov     x0, #imm1
3672 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3673 ;;      cmp     x1, x0
3674 ;;      cset    x2, <ne,eq>
3675 ;; into the shorter:
3676 ;;      sub     x0, x1, #(CST & 0xfff000)
3677 ;;      subs    x0, x0, #(CST & 0x000fff)
3678 ;;      cset x2, <ne, eq>.
3679 (define_insn_and_split "*compare_cstore<mode>_insn"
3680   [(set (match_operand:GPI 0 "register_operand" "=r")
3681          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3682                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3683    (clobber (reg:CC CC_REGNUM))]
3684   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3685    && !aarch64_plus_operand (operands[2], <MODE>mode)
3686    && !reload_completed"
3687   "#"
3688   "&& true"
3689   [(const_int 0)]
3690   {
3691     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3692     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3693     rtx tmp = gen_reg_rtx (<MODE>mode);
3694     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3695     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3696     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3697     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3698     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3699     DONE;
3700   }
3701   [(set_attr "type" "csel")]
3702 )
3703
3704 ;; zero_extend version of the above
3705 (define_insn "*cstoresi_insn_uxtw"
3706   [(set (match_operand:DI 0 "register_operand" "=r")
3707         (zero_extend:DI
3708          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3709           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3710   ""
3711   "cset\\t%w0, %m1"
3712   [(set_attr "type" "csel")]
3713 )
3714
3715 (define_insn "cstore<mode>_neg"
3716   [(set (match_operand:ALLI 0 "register_operand" "=r")
3717         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3718                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
3719   ""
3720   "csetm\\t%<w>0, %m1"
3721   [(set_attr "type" "csel")]
3722 )
3723
3724 ;; zero_extend version of the above
3725 (define_insn "*cstoresi_neg_uxtw"
3726   [(set (match_operand:DI 0 "register_operand" "=r")
3727         (zero_extend:DI
3728          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3729                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3730   ""
3731   "csetm\\t%w0, %m1"
3732   [(set_attr "type" "csel")]
3733 )
3734
3735 (define_expand "cmov<mode>6"
3736   [(set (match_operand:GPI 0 "register_operand" "")
3737         (if_then_else:GPI
3738          (match_operator 1 "aarch64_comparison_operator"
3739           [(match_operand:GPI 2 "register_operand" "")
3740            (match_operand:GPI 3 "aarch64_plus_operand" "")])
3741          (match_operand:GPI 4 "register_operand" "")
3742          (match_operand:GPI 5 "register_operand" "")))]
3743   ""
3744   "
3745   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3746                                       operands[3]);
3747   operands[3] = const0_rtx;
3748   "
3749 )
3750
3751 (define_expand "cmov<mode>6"
3752   [(set (match_operand:GPF 0 "register_operand" "")
3753         (if_then_else:GPF
3754          (match_operator 1 "aarch64_comparison_operator"
3755           [(match_operand:GPF 2 "register_operand" "")
3756            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3757          (match_operand:GPF 4 "register_operand" "")
3758          (match_operand:GPF 5 "register_operand" "")))]
3759   ""
3760   "
3761   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3762                                       operands[3]);
3763   operands[3] = const0_rtx;
3764   "
3765 )
3766
3767 (define_insn "*cmov<mode>_insn"
3768   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3769         (if_then_else:ALLI
3770          (match_operator 1 "aarch64_comparison_operator"
3771           [(match_operand 2 "cc_register" "") (const_int 0)])
3772          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3773          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3774   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3775      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3776   ;; Final two alternatives should be unreachable, but included for completeness
3777   "@
3778    csel\\t%<w>0, %<w>3, %<w>4, %m1
3779    csinv\\t%<w>0, %<w>3, <w>zr, %m1
3780    csinv\\t%<w>0, %<w>4, <w>zr, %M1
3781    csinc\\t%<w>0, %<w>3, <w>zr, %m1
3782    csinc\\t%<w>0, %<w>4, <w>zr, %M1
3783    mov\\t%<w>0, -1
3784    mov\\t%<w>0, 1"
3785   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3786 )
3787
3788 ;; zero_extend version of above
3789 (define_insn "*cmovsi_insn_uxtw"
3790   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3791         (zero_extend:DI
3792          (if_then_else:SI
3793           (match_operator 1 "aarch64_comparison_operator"
3794            [(match_operand 2 "cc_register" "") (const_int 0)])
3795           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3796           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3797   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3798      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3799   ;; Final two alternatives should be unreachable, but included for completeness
3800   "@
3801    csel\\t%w0, %w3, %w4, %m1
3802    csinv\\t%w0, %w3, wzr, %m1
3803    csinv\\t%w0, %w4, wzr, %M1
3804    csinc\\t%w0, %w3, wzr, %m1
3805    csinc\\t%w0, %w4, wzr, %M1
3806    mov\\t%w0, -1
3807    mov\\t%w0, 1"
3808   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3809 )
3810
3811 (define_insn "*cmovdi_insn_uxtw"
3812   [(set (match_operand:DI 0 "register_operand" "=r")
3813         (if_then_else:DI
3814          (match_operator 1 "aarch64_comparison_operator"
3815           [(match_operand 2 "cc_register" "") (const_int 0)])
3816          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3817          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3818   ""
3819   "csel\\t%w0, %w3, %w4, %m1"
3820   [(set_attr "type" "csel")]
3821 )
3822
3823 (define_insn "*cmov<mode>_insn"
3824   [(set (match_operand:GPF 0 "register_operand" "=w")
3825         (if_then_else:GPF
3826          (match_operator 1 "aarch64_comparison_operator"
3827           [(match_operand 2 "cc_register" "") (const_int 0)])
3828          (match_operand:GPF 3 "register_operand" "w")
3829          (match_operand:GPF 4 "register_operand" "w")))]
3830   "TARGET_FLOAT"
3831   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3832   [(set_attr "type" "fcsel")]
3833 )
3834
3835 (define_expand "mov<mode>cc"
3836   [(set (match_operand:ALLI 0 "register_operand" "")
3837         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3838                            (match_operand:ALLI 2 "register_operand" "")
3839                            (match_operand:ALLI 3 "register_operand" "")))]
3840   ""
3841   {
3842     rtx ccreg;
3843     enum rtx_code code = GET_CODE (operands[1]);
3844
3845     if (code == UNEQ || code == LTGT)
3846       FAIL;
3847
3848     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3849                                      XEXP (operands[1], 1));
3850     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3851   }
3852 )
3853
3854 (define_expand "mov<GPF:mode><GPI:mode>cc"
3855   [(set (match_operand:GPI 0 "register_operand" "")
3856         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3857                           (match_operand:GPF 2 "register_operand" "")
3858                           (match_operand:GPF 3 "register_operand" "")))]
3859   ""
3860   {
3861     rtx ccreg;
3862     enum rtx_code code = GET_CODE (operands[1]);
3863
3864     if (code == UNEQ || code == LTGT)
3865       FAIL;
3866
3867     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3868                                   XEXP (operands[1], 1));
3869     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3870   }
3871 )
3872
3873 (define_expand "mov<mode>cc"
3874   [(set (match_operand:GPF 0 "register_operand" "")
3875         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3876                           (match_operand:GPF 2 "register_operand" "")
3877                           (match_operand:GPF 3 "register_operand" "")))]
3878   ""
3879   {
3880     rtx ccreg;
3881     enum rtx_code code = GET_CODE (operands[1]);
3882
3883     if (code == UNEQ || code == LTGT)
3884       FAIL;
3885
3886     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3887                                   XEXP (operands[1], 1));
3888     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3889   }
3890 )
3891
3892 (define_expand "<neg_not_op><mode>cc"
3893   [(set (match_operand:GPI 0 "register_operand" "")
3894         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3895                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3896                           (match_operand:GPI 3 "register_operand" "")))]
3897   ""
3898   {
3899     rtx ccreg;
3900     enum rtx_code code = GET_CODE (operands[1]);
3901
3902     if (code == UNEQ || code == LTGT)
3903       FAIL;
3904
3905     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3906                                       XEXP (operands[1], 1));
3907     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3908   }
3909 )
3910
3911 ;; CRC32 instructions.
3912 (define_insn "aarch64_<crc_variant>"
3913   [(set (match_operand:SI 0 "register_operand" "=r")
3914         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3915                     (match_operand:<crc_mode> 2 "register_operand" "r")]
3916          CRC))]
3917   "TARGET_CRC32"
3918   {
3919     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
3920       return "<crc_variant>\\t%w0, %w1, %x2";
3921     else
3922       return "<crc_variant>\\t%w0, %w1, %w2";
3923   }
3924   [(set_attr "type" "crc")]
3925 )
3926
3927 (define_insn "*csinc2<mode>_insn"
3928   [(set (match_operand:GPI 0 "register_operand" "=r")
3929         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3930                   (match_operand:GPI 1 "register_operand" "r")))]
3931   ""
3932   "cinc\\t%<w>0, %<w>1, %m2"
3933   [(set_attr "type" "csel")]
3934 )
3935
3936 (define_insn "csinc3<mode>_insn"
3937   [(set (match_operand:GPI 0 "register_operand" "=r")
3938         (if_then_else:GPI
3939           (match_operand 1 "aarch64_comparison_operation" "")
3940           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3941                     (const_int 1))
3942           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3943   ""
3944   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3945   [(set_attr "type" "csel")]
3946 )
3947
3948 (define_insn "*csinv3<mode>_insn"
3949   [(set (match_operand:GPI 0 "register_operand" "=r")
3950         (if_then_else:GPI
3951           (match_operand 1 "aarch64_comparison_operation" "")
3952           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3953           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3954   ""
3955   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3956   [(set_attr "type" "csel")]
3957 )
3958
3959 (define_insn "csneg3_uxtw_insn"
3960   [(set (match_operand:DI 0 "register_operand" "=r")
3961         (zero_extend:DI
3962           (if_then_else:SI
3963             (match_operand 1 "aarch64_comparison_operation" "")
3964             (neg:SI (match_operand:SI 2 "register_operand" "r"))
3965             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3966   ""
3967   "csneg\\t%w0, %w3, %w2, %M1"
3968   [(set_attr "type" "csel")]
3969 )
3970
3971 (define_insn "csneg3<mode>_insn"
3972   [(set (match_operand:GPI 0 "register_operand" "=r")
3973         (if_then_else:GPI
3974           (match_operand 1 "aarch64_comparison_operation" "")
3975           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3976           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3977   ""
3978   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3979   [(set_attr "type" "csel")]
3980 )
3981
3982 ;; If X can be loaded by a single CNT[BHWD] instruction,
3983 ;;
3984 ;;    A = UMAX (B, X)
3985 ;;
3986 ;; is equivalent to:
3987 ;;
3988 ;;    TMP = UQDEC[BHWD] (B, X)
3989 ;;    A = TMP + X
3990 ;;
3991 ;; Defining the pattern this way means that:
3992 ;;
3993 ;;    A = UMAX (B, X) - X
3994 ;;
3995 ;; becomes:
3996 ;;
3997 ;;    TMP1 = UQDEC[BHWD] (B, X)
3998 ;;    TMP2 = TMP1 + X
3999 ;;    A = TMP2 - X
4000 ;;
4001 ;; which combine can optimize to:
4002 ;;
4003 ;;    A = UQDEC[BHWD] (B, X)
4004 ;;
4005 ;; We don't use match_operand predicates because the order of the operands
4006 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4007 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4008 (define_expand "umax<mode>3"
4009   [(set (match_operand:GPI 0 "register_operand")
4010         (umax:GPI (match_operand:GPI 1 "")
4011                   (match_operand:GPI 2 "")))]
4012   "TARGET_SVE"
4013   {
4014     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4015       std::swap (operands[1], operands[2]);
4016     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4017       FAIL;
4018     rtx temp = gen_reg_rtx (<MODE>mode);
4019     operands[1] = force_reg (<MODE>mode, operands[1]);
4020     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4021     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4022     DONE;
4023   }
4024 )
4025
4026 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4027 (define_insn "aarch64_uqdec<mode>"
4028   [(set (match_operand:GPI 0 "register_operand" "=r")
4029         (minus:GPI
4030          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4031                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4032          (match_dup 2)))]
4033   "TARGET_SVE"
4034   {
4035     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4036   }
4037 )
4038
4039 ;; -------------------------------------------------------------------
4040 ;; Logical operations
4041 ;; -------------------------------------------------------------------
4042
4043
4044 (define_insn_and_split "*aarch64_and<mode>_imm2"
4045   [(set (match_operand:GPI 0 "register_operand" "=rk")
4046         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4047                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4048   ""
4049   "#"
4050   "true"
4051   [(const_int 0)]
4052   {
4053      HOST_WIDE_INT val = INTVAL (operands[2]);
4054      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4055      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4056
4057      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4058      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4059      DONE;
4060   }
4061 )
4062
4063 (define_insn "<optab><mode>3"
4064   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4065         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4066                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4067   ""
4068   "@
4069   <logical>\\t%<w>0, %<w>1, %<w>2
4070   <logical>\\t%<w>0, %<w>1, %2
4071   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4072   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4073    (set_attr "simd" "*,*,yes")]
4074 )
4075
4076 ;; zero_extend version of above
4077 (define_insn "*<optab>si3_uxtw"
4078   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4079         (zero_extend:DI
4080          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4081                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4082   ""
4083   "@
4084    <logical>\\t%w0, %w1, %w2
4085    <logical>\\t%w0, %w1, %2"
4086   [(set_attr "type" "logic_reg,logic_imm")]
4087 )
4088
4089 (define_insn "*and<mode>3_compare0"
4090   [(set (reg:CC_NZ CC_REGNUM)
4091         (compare:CC_NZ
4092          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4093                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4094          (const_int 0)))
4095    (set (match_operand:GPI 0 "register_operand" "=r,r")
4096         (and:GPI (match_dup 1) (match_dup 2)))]
4097   ""
4098   "@
4099    ands\\t%<w>0, %<w>1, %<w>2
4100    ands\\t%<w>0, %<w>1, %2"
4101   [(set_attr "type" "logics_reg,logics_imm")]
4102 )
4103
4104 ;; zero_extend version of above
4105 (define_insn "*andsi3_compare0_uxtw"
4106   [(set (reg:CC_NZ CC_REGNUM)
4107         (compare:CC_NZ
4108          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4109                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4110          (const_int 0)))
4111    (set (match_operand:DI 0 "register_operand" "=r,r")
4112         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4113   ""
4114   "@
4115    ands\\t%w0, %w1, %w2
4116    ands\\t%w0, %w1, %2"
4117   [(set_attr "type" "logics_reg,logics_imm")]
4118 )
4119
4120 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4121   [(set (reg:CC_NZ CC_REGNUM)
4122         (compare:CC_NZ
4123          (and:GPI (SHIFT:GPI
4124                    (match_operand:GPI 1 "register_operand" "r")
4125                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4126                   (match_operand:GPI 3 "register_operand" "r"))
4127          (const_int 0)))
4128    (set (match_operand:GPI 0 "register_operand" "=r")
4129         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4130   ""
4131   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4132   [(set_attr "type" "logics_shift_imm")]
4133 )
4134
4135 ;; zero_extend version of above
4136 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4137   [(set (reg:CC_NZ CC_REGNUM)
4138         (compare:CC_NZ
4139          (and:SI (SHIFT:SI
4140                   (match_operand:SI 1 "register_operand" "r")
4141                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4142                  (match_operand:SI 3 "register_operand" "r"))
4143          (const_int 0)))
4144    (set (match_operand:DI 0 "register_operand" "=r")
4145         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4146                                 (match_dup 3))))]
4147   ""
4148   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4149   [(set_attr "type" "logics_shift_imm")]
4150 )
4151
4152 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4153   [(set (match_operand:GPI 0 "register_operand" "=r")
4154         (LOGICAL:GPI (SHIFT:GPI
4155                       (match_operand:GPI 1 "register_operand" "r")
4156                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4157                      (match_operand:GPI 3 "register_operand" "r")))]
4158   ""
4159   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4160   [(set_attr "type" "logic_shift_imm")]
4161 )
4162
4163 (define_insn "*<optab>_rol<mode>3"
4164   [(set (match_operand:GPI 0 "register_operand" "=r")
4165         (LOGICAL:GPI (rotate:GPI
4166                       (match_operand:GPI 1 "register_operand" "r")
4167                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4168                      (match_operand:GPI 3 "register_operand" "r")))]
4169   ""
4170   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4171   [(set_attr "type" "logic_shift_imm")]
4172 )
4173
4174 ;; zero_extend versions of above
4175 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4176   [(set (match_operand:DI 0 "register_operand" "=r")
4177         (zero_extend:DI
4178          (LOGICAL:SI (SHIFT:SI
4179                       (match_operand:SI 1 "register_operand" "r")
4180                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4181                      (match_operand:SI 3 "register_operand" "r"))))]
4182   ""
4183   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4184   [(set_attr "type" "logic_shift_imm")]
4185 )
4186
4187 (define_insn "*<optab>_rolsi3_uxtw"
4188   [(set (match_operand:DI 0 "register_operand" "=r")
4189         (zero_extend:DI
4190          (LOGICAL:SI (rotate:SI
4191                       (match_operand:SI 1 "register_operand" "r")
4192                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4193                      (match_operand:SI 3 "register_operand" "r"))))]
4194   ""
4195   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4196   [(set_attr "type" "logic_shift_imm")]
4197 )
4198
4199 (define_insn "one_cmpl<mode>2"
4200   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4201         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4202   ""
4203   "@
4204   mvn\\t%<w>0, %<w>1
4205   mvn\\t%0.8b, %1.8b"
4206   [(set_attr "type" "logic_reg,neon_logic")
4207    (set_attr "simd" "*,yes")]
4208 )
4209
4210 (define_insn "*one_cmpl_<optab><mode>2"
4211   [(set (match_operand:GPI 0 "register_operand" "=r")
4212         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4213                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4214   ""
4215   "mvn\\t%<w>0, %<w>1, <shift> %2"
4216   [(set_attr "type" "logic_shift_imm")]
4217 )
4218
4219 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4220
4221 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4222   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4223         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4224                      (match_operand:GPI 2 "register_operand" "r,w")))]
4225   ""
4226   "@
4227   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4228   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4229   [(set_attr "type" "logic_reg,neon_logic")
4230    (set_attr "simd" "*,yes")]
4231 )
4232
4233 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4234   [(set (match_operand:DI 0 "register_operand" "=r")
4235         (zero_extend:DI
4236           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4237                        (match_operand:SI 2 "register_operand" "r"))))]
4238   ""
4239   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4240   [(set_attr "type" "logic_reg")]
4241 )
4242
4243 (define_insn "*xor_one_cmplsidi3_ze"
4244   [(set (match_operand:DI 0 "register_operand" "=r")
4245         (zero_extend:DI
4246           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4247                           (match_operand:SI 2 "register_operand" "r")))))]
4248   ""
4249   "eon\\t%w0, %w1, %w2"
4250   [(set_attr "type" "logic_reg")]
4251 )
4252
4253 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4254 ;; eon does not operate on SIMD registers so the vector variant must be split.
4255 (define_insn_and_split "*xor_one_cmpl<mode>3"
4256   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4257         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4258                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4259   ""
4260   "@
4261   eon\\t%<w>0, %<w>1, %<w>2
4262   #"
4263   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4264   [(set (match_operand:GPI 0 "register_operand" "=w")
4265         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4266                  (match_operand:GPI 2 "register_operand" "w")))
4267    (set (match_dup 0) (not:GPI (match_dup 0)))]
4268   ""
4269   [(set_attr "type" "logic_reg,multiple")
4270    (set_attr "simd" "*,yes")]
4271 )
4272
4273 (define_insn "*and_one_cmpl<mode>3_compare0"
4274   [(set (reg:CC_NZ CC_REGNUM)
4275         (compare:CC_NZ
4276          (and:GPI (not:GPI
4277                    (match_operand:GPI 1 "register_operand" "r"))
4278                   (match_operand:GPI 2 "register_operand" "r"))
4279          (const_int 0)))
4280    (set (match_operand:GPI 0 "register_operand" "=r")
4281         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4282   ""
4283   "bics\\t%<w>0, %<w>2, %<w>1"
4284   [(set_attr "type" "logics_reg")]
4285 )
4286
4287 ;; zero_extend version of above
4288 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4289   [(set (reg:CC_NZ CC_REGNUM)
4290         (compare:CC_NZ
4291          (and:SI (not:SI
4292                   (match_operand:SI 1 "register_operand" "r"))
4293                  (match_operand:SI 2 "register_operand" "r"))
4294          (const_int 0)))
4295    (set (match_operand:DI 0 "register_operand" "=r")
4296         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4297   ""
4298   "bics\\t%w0, %w2, %w1"
4299   [(set_attr "type" "logics_reg")]
4300 )
4301
4302 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4303   [(set (reg:CC_NZ CC_REGNUM)
4304     (compare:CC_NZ
4305      (and:GPI (not:GPI
4306            (match_operand:GPI 0 "register_operand" "r"))
4307           (match_operand:GPI 1 "register_operand" "r"))
4308      (const_int 0)))]
4309   ""
4310   "bics\\t<w>zr, %<w>1, %<w>0"
4311   [(set_attr "type" "logics_reg")]
4312 )
4313
4314 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4315   [(set (match_operand:GPI 0 "register_operand" "=r")
4316         (LOGICAL:GPI (not:GPI
4317                       (SHIFT:GPI
4318                        (match_operand:GPI 1 "register_operand" "r")
4319                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4320                      (match_operand:GPI 3 "register_operand" "r")))]
4321   ""
4322   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4323   [(set_attr "type" "logic_shift_imm")]
4324 )
4325
4326 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4327   [(set (match_operand:GPI 0 "register_operand" "=r")
4328         (not:GPI (xor:GPI
4329                       (SHIFT:GPI
4330                        (match_operand:GPI 1 "register_operand" "r")
4331                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4332                      (match_operand:GPI 3 "register_operand" "r"))))]
4333   ""
4334   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4335   [(set_attr "type" "logic_shift_imm")]
4336 )
4337
4338 ;; Zero-extend version of the above.
4339 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4340   [(set (match_operand:DI 0 "register_operand" "=r")
4341         (zero_extend:DI
4342           (not:SI (xor:SI
4343                     (SHIFT:SI
4344                       (match_operand:SI 1 "register_operand" "r")
4345                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4346                     (match_operand:SI 3 "register_operand" "r")))))]
4347   ""
4348   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4349   [(set_attr "type" "logic_shift_imm")]
4350 )
4351
4352 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4353   [(set (reg:CC_NZ CC_REGNUM)
4354         (compare:CC_NZ
4355          (and:GPI (not:GPI
4356                    (SHIFT:GPI
4357                     (match_operand:GPI 1 "register_operand" "r")
4358                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4359                   (match_operand:GPI 3 "register_operand" "r"))
4360          (const_int 0)))
4361    (set (match_operand:GPI 0 "register_operand" "=r")
4362         (and:GPI (not:GPI
4363                   (SHIFT:GPI
4364                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4365   ""
4366   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4367   [(set_attr "type" "logics_shift_imm")]
4368 )
4369
4370 ;; zero_extend version of above
4371 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4372   [(set (reg:CC_NZ CC_REGNUM)
4373         (compare:CC_NZ
4374          (and:SI (not:SI
4375                   (SHIFT:SI
4376                    (match_operand:SI 1 "register_operand" "r")
4377                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4378                  (match_operand:SI 3 "register_operand" "r"))
4379          (const_int 0)))
4380    (set (match_operand:DI 0 "register_operand" "=r")
4381         (zero_extend:DI (and:SI
4382                          (not:SI
4383                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4384   ""
4385   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4386   [(set_attr "type" "logics_shift_imm")]
4387 )
4388
4389 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4390   [(set (reg:CC_NZ CC_REGNUM)
4391     (compare:CC_NZ
4392      (and:GPI (not:GPI
4393            (SHIFT:GPI
4394             (match_operand:GPI 0 "register_operand" "r")
4395             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4396           (match_operand:GPI 2 "register_operand" "r"))
4397      (const_int 0)))]
4398   ""
4399   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4400   [(set_attr "type" "logics_shift_imm")]
4401 )
4402
4403 (define_insn "clz<mode>2"
4404   [(set (match_operand:GPI 0 "register_operand" "=r")
4405         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4406   ""
4407   "clz\\t%<w>0, %<w>1"
4408   [(set_attr "type" "clz")]
4409 )
4410
4411 (define_expand "ffs<mode>2"
4412   [(match_operand:GPI 0 "register_operand")
4413    (match_operand:GPI 1 "register_operand")]
4414   ""
4415   {
4416     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4417     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4418
4419     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4420     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4421     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4422     DONE;
4423   }
4424 )
4425
4426 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4427 ;;
4428 ;; MOV  v.1d, x0
4429 ;; CNT  v1.8b, v.8b
4430 ;; ADDV b2, v1.8b
4431 ;; MOV  w0, v2.b[0]
4432
4433 (define_expand "popcount<mode>2"
4434   [(match_operand:GPI 0 "register_operand")
4435    (match_operand:GPI 1 "register_operand")]
4436   "TARGET_SIMD"
4437 {
4438   rtx v = gen_reg_rtx (V8QImode);
4439   rtx v1 = gen_reg_rtx (V8QImode);
4440   rtx r = gen_reg_rtx (QImode);
4441   rtx in = operands[1];
4442   rtx out = operands[0];
4443   if(<MODE>mode == SImode)
4444     {
4445       rtx tmp;
4446       tmp = gen_reg_rtx (DImode);
4447       /* If we have SImode, zero extend to DImode, pop count does
4448          not change if we have extra zeros. */
4449       emit_insn (gen_zero_extendsidi2 (tmp, in));
4450       in = tmp;
4451     }
4452   emit_move_insn (v, gen_lowpart (V8QImode, in));
4453   emit_insn (gen_popcountv8qi2 (v1, v));
4454   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4455   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4456   DONE;
4457 })
4458
4459 (define_insn "clrsb<mode>2"
4460   [(set (match_operand:GPI 0 "register_operand" "=r")
4461         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4462   ""
4463   "cls\\t%<w>0, %<w>1"
4464   [(set_attr "type" "clz")]
4465 )
4466
4467 (define_insn "rbit<mode>2"
4468   [(set (match_operand:GPI 0 "register_operand" "=r")
4469         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4470   ""
4471   "rbit\\t%<w>0, %<w>1"
4472   [(set_attr "type" "rbit")]
4473 )
4474
4475 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4476 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4477 ;; expression and split after reload to enable scheduling them apart if
4478 ;; needed.
4479
4480 (define_insn_and_split "ctz<mode>2"
4481  [(set (match_operand:GPI           0 "register_operand" "=r")
4482        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4483   ""
4484   "#"
4485   "reload_completed"
4486   [(const_int 0)]
4487   "
4488   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4489   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4490   DONE;
4491 ")
4492
4493 (define_insn "*and<mode>_compare0"
4494   [(set (reg:CC_NZ CC_REGNUM)
4495         (compare:CC_NZ
4496          (match_operand:SHORT 0 "register_operand" "r")
4497          (const_int 0)))]
4498   ""
4499   "tst\\t%<w>0, <short_mask>"
4500   [(set_attr "type" "alus_imm")]
4501 )
4502
4503 (define_insn "*ands<mode>_compare0"
4504   [(set (reg:CC_NZ CC_REGNUM)
4505         (compare:CC_NZ
4506          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4507          (const_int 0)))
4508    (set (match_operand:GPI 0 "register_operand" "=r")
4509         (zero_extend:GPI (match_dup 1)))]
4510   ""
4511   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4512   [(set_attr "type" "alus_imm")]
4513 )
4514
4515 (define_insn "*and<mode>3nr_compare0"
4516   [(set (reg:CC_NZ CC_REGNUM)
4517         (compare:CC_NZ
4518          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4519                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4520          (const_int 0)))]
4521   ""
4522   "@
4523    tst\\t%<w>0, %<w>1
4524    tst\\t%<w>0, %1"
4525   [(set_attr "type" "logics_reg,logics_imm")]
4526 )
4527
4528 (define_split
4529   [(set (reg:CC_NZ CC_REGNUM)
4530         (compare:CC_NZ
4531          (and:GPI (match_operand:GPI 0 "register_operand")
4532                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4533          (const_int 0)))
4534    (clobber (match_operand:SI 2 "register_operand"))]
4535   ""
4536   [(set (match_dup 2) (match_dup 1))
4537    (set (reg:CC_NZ CC_REGNUM)
4538         (compare:CC_NZ
4539          (and:GPI (match_dup 0)
4540                   (match_dup 2))
4541          (const_int 0)))]
4542 )
4543
4544 (define_insn "*and<mode>3nr_compare0_zextract"
4545   [(set (reg:CC_NZ CC_REGNUM)
4546         (compare:CC_NZ
4547          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4548                   (match_operand:GPI 1 "const_int_operand" "n")
4549                   (match_operand:GPI 2 "const_int_operand" "n"))
4550          (const_int 0)))]
4551   "INTVAL (operands[1]) > 0
4552    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4553         <= GET_MODE_BITSIZE (<MODE>mode))
4554    && aarch64_bitmask_imm (
4555         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4556                                                  operands[2])),
4557         <MODE>mode)"
4558   {
4559     operands[1]
4560       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4561     return "tst\\t%<w>0, %1";
4562   }
4563   [(set_attr "type" "logics_shift_imm")]
4564 )
4565
4566 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4567   [(set (reg:CC_NZ CC_REGNUM)
4568         (compare:CC_NZ
4569          (and:GPI (SHIFT:GPI
4570                    (match_operand:GPI 0 "register_operand" "r")
4571                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4572                   (match_operand:GPI 2 "register_operand" "r"))
4573         (const_int 0)))]
4574   ""
4575   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4576   [(set_attr "type" "logics_shift_imm")]
4577 )
4578
4579 (define_split
4580   [(set (reg:CC_NZ CC_REGNUM)
4581         (compare:CC_NZ
4582          (and:GPI (SHIFT:GPI
4583                    (match_operand:GPI 0 "register_operand")
4584                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4585                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4586         (const_int 0)))
4587     (clobber (match_operand:SI 3 "register_operand"))]
4588   ""
4589   [(set (match_dup 3) (match_dup 2))
4590    (set (reg:CC_NZ CC_REGNUM)
4591         (compare:CC_NZ
4592          (and:GPI (SHIFT:GPI
4593                    (match_dup 0)
4594                    (match_dup 1))
4595                   (match_dup 3))
4596          (const_int 0)))]
4597 )
4598
4599 ;; -------------------------------------------------------------------
4600 ;; Shifts
4601 ;; -------------------------------------------------------------------
4602
4603 (define_expand "<optab><mode>3"
4604   [(set (match_operand:GPI 0 "register_operand")
4605         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4606                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4607   ""
4608   {
4609     if (CONST_INT_P (operands[2]))
4610       {
4611         operands[2] = GEN_INT (INTVAL (operands[2])
4612                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4613
4614         if (operands[2] == const0_rtx)
4615           {
4616             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4617             DONE;
4618           }
4619       }
4620   }
4621 )
4622
4623 (define_expand "ashl<mode>3"
4624   [(set (match_operand:SHORT 0 "register_operand")
4625         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4626                       (match_operand:QI 2 "const_int_operand")))]
4627   ""
4628   {
4629     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4630
4631     if (operands[2] == const0_rtx)
4632       {
4633         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4634         DONE;
4635       }
4636   }
4637 )
4638
4639 (define_expand "rotr<mode>3"
4640   [(set (match_operand:GPI 0 "register_operand")
4641         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4642                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4643   ""
4644   {
4645     if (CONST_INT_P (operands[2]))
4646       {
4647         operands[2] = GEN_INT (INTVAL (operands[2])
4648                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4649
4650         if (operands[2] == const0_rtx)
4651           {
4652             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4653             DONE;
4654           }
4655       }
4656   }
4657 )
4658
4659 (define_expand "rotl<mode>3"
4660   [(set (match_operand:GPI 0 "register_operand")
4661         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4662                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4663   ""
4664   {
4665     /* (SZ - cnt) % SZ == -cnt % SZ */
4666     if (CONST_INT_P (operands[2]))
4667       {
4668         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4669                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4670         if (operands[2] == const0_rtx)
4671           {
4672             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4673             DONE;
4674           }
4675       }
4676     else
4677       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4678                                         NULL_RTX, 1);
4679   }
4680 )
4681
4682 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4683 ;; they truncate the shift/rotate amount by the size of the registers they
4684 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
4685 ;; such redundant masking instructions.  GCC can do that automatically when
4686 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4687 ;; because some of the SISD shift alternatives don't perform this truncations.
4688 ;; So this pattern exists to catch such cases.
4689
4690 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4691   [(set (match_operand:GPI 0 "register_operand" "=r")
4692         (SHIFT:GPI
4693           (match_operand:GPI 1 "register_operand" "r")
4694           (match_operator 4 "subreg_lowpart_operator"
4695            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4696                      (match_operand 3 "const_int_operand" "n"))])))]
4697   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4698   "<shift>\t%<w>0, %<w>1, %<w>2"
4699   [(set_attr "type" "shift_reg")]
4700 )
4701
4702 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4703   [(set (match_operand:GPI 0 "register_operand" "=&r")
4704         (SHIFT:GPI
4705           (match_operand:GPI 1 "register_operand" "r")
4706           (match_operator 4 "subreg_lowpart_operator"
4707           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4708                            (match_operand 3 "const_int_operand" "n")))])))]
4709   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4710   "#"
4711   "&& true"
4712   [(const_int 0)]
4713   {
4714     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4715                : lowpart_subreg (SImode, operands[0], <MODE>mode));
4716     emit_insn (gen_negsi2 (tmp, operands[2]));
4717
4718     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4719     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4720                                      SUBREG_BYTE (operands[4]));
4721     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4722     DONE;
4723   }
4724 )
4725
4726 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
4727   [(set (match_operand:GPI 0 "register_operand" "=&r")
4728         (ashift:GPI
4729           (match_operand:GPI 1 "register_operand" "r")
4730           (minus:QI (match_operand 2 "const_int_operand" "n")
4731                     (match_operator 5 "subreg_lowpart_operator"
4732                     [(and:SI (match_operand:SI 3 "register_operand" "r")
4733                              (match_operand 4 "const_int_operand" "n"))]))))]
4734   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4735    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4736   "#"
4737   "&& true"
4738   [(const_int 0)]
4739   {
4740     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4741                : operands[0]);
4742
4743     emit_insn (gen_negsi2 (tmp, operands[3]));
4744
4745     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
4746     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
4747                                      SUBREG_BYTE (operands[5]));
4748
4749     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
4750     DONE;
4751   }
4752 )
4753
4754 (define_insn "*aarch64_<optab>_reg_di3_mask2"
4755   [(set (match_operand:DI 0 "register_operand" "=r")
4756         (SHIFT:DI
4757           (match_operand:DI 1 "register_operand" "r")
4758           (match_operator 4 "subreg_lowpart_operator"
4759            [(and:SI (match_operand:SI 2 "register_operand" "r")
4760                     (match_operand 3 "const_int_operand" "n"))])))]
4761   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
4762 {
4763   rtx xop[3];
4764   xop[0] = operands[0];
4765   xop[1] = operands[1];
4766   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
4767   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
4768   return "";
4769 }
4770   [(set_attr "type" "shift_reg")]
4771 )
4772
4773 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
4774   [(set (match_operand:GPI 0 "register_operand" "=&r")
4775         (ASHIFT:GPI
4776           (match_operand:GPI 1 "register_operand" "r")
4777           (minus:QI (match_operand 2 "const_int_operand" "n")
4778                     (match_operand:QI 3 "register_operand" "r"))))]
4779   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4780   "#"
4781   "&& true"
4782   [(const_int 0)]
4783   {
4784     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
4785
4786     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4787                : gen_lowpart (SImode, operands[0]));
4788
4789     emit_insn (gen_negsi2 (tmp, subreg_tmp));
4790
4791     rtx and_op = gen_rtx_AND (SImode, tmp,
4792                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
4793
4794     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
4795
4796     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
4797     DONE;
4798   }
4799   [(set_attr "length" "8")]
4800 )
4801
4802 ;; Logical left shift using SISD or Integer instruction
4803 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
4804   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
4805         (ashift:GPI
4806           (match_operand:GPI 1 "register_operand" "r,r,w,w")
4807           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
4808   ""
4809   "@
4810    lsl\t%<w>0, %<w>1, %2
4811    lsl\t%<w>0, %<w>1, %<w>2
4812    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4813    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
4814   [(set_attr "simd" "no,no,yes,yes")
4815    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
4816 )
4817
4818 ;; Logical right shift using SISD or Integer instruction
4819 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
4820   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4821         (lshiftrt:GPI
4822          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4823          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
4824                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4825   ""
4826   "@
4827    lsr\t%<w>0, %<w>1, %2
4828    lsr\t%<w>0, %<w>1, %<w>2
4829    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4830    #
4831    #"
4832   [(set_attr "simd" "no,no,yes,yes,yes")
4833    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4834 )
4835
4836 (define_split
4837   [(set (match_operand:DI 0 "aarch64_simd_register")
4838         (lshiftrt:DI
4839            (match_operand:DI 1 "aarch64_simd_register")
4840            (match_operand:QI 2 "aarch64_simd_register")))]
4841   "TARGET_SIMD && reload_completed"
4842   [(set (match_dup 3)
4843         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4844    (set (match_dup 0)
4845         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4846   {
4847     operands[3] = gen_lowpart (QImode, operands[0]);
4848   }
4849 )
4850
4851 (define_split
4852   [(set (match_operand:SI 0 "aarch64_simd_register")
4853         (lshiftrt:SI
4854            (match_operand:SI 1 "aarch64_simd_register")
4855            (match_operand:QI 2 "aarch64_simd_register")))]
4856   "TARGET_SIMD && reload_completed"
4857   [(set (match_dup 3)
4858         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4859    (set (match_dup 0)
4860         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4861   {
4862     operands[3] = gen_lowpart (QImode, operands[0]);
4863   }
4864 )
4865
4866 ;; Arithmetic right shift using SISD or Integer instruction
4867 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4868   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4869         (ashiftrt:GPI
4870           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4871           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
4872                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4873   ""
4874   "@
4875    asr\t%<w>0, %<w>1, %2
4876    asr\t%<w>0, %<w>1, %<w>2
4877    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4878    #
4879    #"
4880   [(set_attr "simd" "no,no,yes,yes,yes")
4881    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4882 )
4883
4884 (define_split
4885   [(set (match_operand:DI 0 "aarch64_simd_register")
4886         (ashiftrt:DI
4887            (match_operand:DI 1 "aarch64_simd_register")
4888            (match_operand:QI 2 "aarch64_simd_register")))]
4889   "TARGET_SIMD && reload_completed"
4890   [(set (match_dup 3)
4891         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4892    (set (match_dup 0)
4893         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4894 {
4895   operands[3] = gen_lowpart (QImode, operands[0]);
4896 }
4897 )
4898
4899 (define_split
4900   [(set (match_operand:SI 0 "aarch64_simd_register")
4901         (ashiftrt:SI
4902            (match_operand:SI 1 "aarch64_simd_register")
4903            (match_operand:QI 2 "aarch64_simd_register")))]
4904   "TARGET_SIMD && reload_completed"
4905   [(set (match_dup 3)
4906         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4907    (set (match_dup 0)
4908         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4909 {
4910   operands[3] = gen_lowpart (QImode, operands[0]);
4911 }
4912 )
4913
4914 (define_insn "*aarch64_sisd_ushl"
4915   [(set (match_operand:DI 0 "register_operand" "=w")
4916         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4917                     (match_operand:QI 2 "register_operand" "w")]
4918                    UNSPEC_SISD_USHL))]
4919   "TARGET_SIMD"
4920   "ushl\t%d0, %d1, %d2"
4921   [(set_attr "simd" "yes")
4922    (set_attr "type" "neon_shift_reg")]
4923 )
4924
4925 (define_insn "*aarch64_ushl_2s"
4926   [(set (match_operand:SI 0 "register_operand" "=w")
4927         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4928                     (match_operand:QI 2 "register_operand" "w")]
4929                    UNSPEC_USHL_2S))]
4930   "TARGET_SIMD"
4931   "ushl\t%0.2s, %1.2s, %2.2s"
4932   [(set_attr "simd" "yes")
4933    (set_attr "type" "neon_shift_reg")]
4934 )
4935
4936 (define_insn "*aarch64_sisd_sshl"
4937   [(set (match_operand:DI 0 "register_operand" "=w")
4938         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4939                     (match_operand:QI 2 "register_operand" "w")]
4940                    UNSPEC_SISD_SSHL))]
4941   "TARGET_SIMD"
4942   "sshl\t%d0, %d1, %d2"
4943   [(set_attr "simd" "yes")
4944    (set_attr "type" "neon_shift_reg")]
4945 )
4946
4947 (define_insn "*aarch64_sshl_2s"
4948   [(set (match_operand:SI 0 "register_operand" "=w")
4949         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4950                     (match_operand:QI 2 "register_operand" "w")]
4951                    UNSPEC_SSHL_2S))]
4952   "TARGET_SIMD"
4953   "sshl\t%0.2s, %1.2s, %2.2s"
4954   [(set_attr "simd" "yes")
4955    (set_attr "type" "neon_shift_reg")]
4956 )
4957
4958 (define_insn "*aarch64_sisd_neg_qi"
4959   [(set (match_operand:QI 0 "register_operand" "=w")
4960         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4961                    UNSPEC_SISD_NEG))]
4962   "TARGET_SIMD"
4963   "neg\t%d0, %d1"
4964   [(set_attr "simd" "yes")
4965    (set_attr "type" "neon_neg")]
4966 )
4967
4968 ;; Rotate right
4969 (define_insn "*ror<mode>3_insn"
4970   [(set (match_operand:GPI 0 "register_operand" "=r,r")
4971      (rotatert:GPI
4972        (match_operand:GPI 1 "register_operand" "r,r")
4973        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
4974   ""
4975   "@
4976    ror\\t%<w>0, %<w>1, %2
4977    ror\\t%<w>0, %<w>1, %<w>2"
4978   [(set_attr "type" "rotate_imm,shift_reg")]
4979 )
4980
4981 ;; zero_extend version of above
4982 (define_insn "*<optab>si3_insn_uxtw"
4983   [(set (match_operand:DI 0 "register_operand" "=r,r")
4984         (zero_extend:DI (SHIFT:SI
4985          (match_operand:SI 1 "register_operand" "r,r")
4986          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
4987   ""
4988   "@
4989    <shift>\\t%w0, %w1, %2
4990    <shift>\\t%w0, %w1, %w2"
4991   [(set_attr "type" "bfx,shift_reg")]
4992 )
4993
4994 (define_insn "*<optab><mode>3_insn"
4995   [(set (match_operand:SHORT 0 "register_operand" "=r")
4996         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
4997                       (match_operand 2 "const_int_operand" "n")))]
4998   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4999 {
5000   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5001   return "<bfshift>\t%w0, %w1, %2, %3";
5002 }
5003   [(set_attr "type" "bfx")]
5004 )
5005
5006 (define_insn "*extr<mode>5_insn"
5007   [(set (match_operand:GPI 0 "register_operand" "=r")
5008         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5009                              (match_operand 3 "const_int_operand" "n"))
5010                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5011                                (match_operand 4 "const_int_operand" "n"))))]
5012   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5013    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5014   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5015   [(set_attr "type" "rotate_imm")]
5016 )
5017
5018 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5019 ;; so we have to match both orderings.
5020 (define_insn "*extr<mode>5_insn_alt"
5021   [(set (match_operand:GPI 0 "register_operand" "=r")
5022         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5023                                 (match_operand 4 "const_int_operand" "n"))
5024                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5025                               (match_operand 3 "const_int_operand" "n"))))]
5026   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5027    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5028        == GET_MODE_BITSIZE (<MODE>mode))"
5029   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5030   [(set_attr "type" "rotate_imm")]
5031 )
5032
5033 ;; zero_extend version of the above
5034 (define_insn "*extrsi5_insn_uxtw"
5035   [(set (match_operand:DI 0 "register_operand" "=r")
5036         (zero_extend:DI
5037          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5038                             (match_operand 3 "const_int_operand" "n"))
5039                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5040                               (match_operand 4 "const_int_operand" "n")))))]
5041   "UINTVAL (operands[3]) < 32 &&
5042    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5043   "extr\\t%w0, %w1, %w2, %4"
5044   [(set_attr "type" "rotate_imm")]
5045 )
5046
5047 (define_insn "*extrsi5_insn_uxtw_alt"
5048   [(set (match_operand:DI 0 "register_operand" "=r")
5049         (zero_extend:DI
5050          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5051                                (match_operand 4 "const_int_operand" "n"))
5052                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5053                             (match_operand 3 "const_int_operand" "n")))))]
5054   "UINTVAL (operands[3]) < 32 &&
5055    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5056   "extr\\t%w0, %w1, %w2, %4"
5057   [(set_attr "type" "rotate_imm")]
5058 )
5059
5060 (define_insn "*ror<mode>3_insn"
5061   [(set (match_operand:GPI 0 "register_operand" "=r")
5062         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5063                     (match_operand 2 "const_int_operand" "n")))]
5064   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5065 {
5066   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5067   return "ror\\t%<w>0, %<w>1, %3";
5068 }
5069   [(set_attr "type" "rotate_imm")]
5070 )
5071
5072 ;; zero_extend version of the above
5073 (define_insn "*rorsi3_insn_uxtw"
5074   [(set (match_operand:DI 0 "register_operand" "=r")
5075         (zero_extend:DI
5076          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5077                     (match_operand 2 "const_int_operand" "n"))))]
5078   "UINTVAL (operands[2]) < 32"
5079 {
5080   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5081   return "ror\\t%w0, %w1, %3";
5082 }
5083   [(set_attr "type" "rotate_imm")]
5084 )
5085
5086 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5087   [(set (match_operand:GPI 0 "register_operand" "=r")
5088         (ANY_EXTEND:GPI
5089          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5090                        (match_operand 2 "const_int_operand" "n"))))]
5091   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5092 {
5093   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5094   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5095 }
5096   [(set_attr "type" "bfx")]
5097 )
5098
5099 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5100   [(set (match_operand:GPI 0 "register_operand" "=r")
5101         (zero_extend:GPI
5102          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5103                          (match_operand 2 "const_int_operand" "n"))))]
5104   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5105 {
5106   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5107   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5108 }
5109   [(set_attr "type" "bfx")]
5110 )
5111
5112 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5113   [(set (match_operand:GPI 0 "register_operand" "=r")
5114         (sign_extend:GPI
5115          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5116                          (match_operand 2 "const_int_operand" "n"))))]
5117   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5118 {
5119   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5120   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5121 }
5122   [(set_attr "type" "bfx")]
5123 )
5124
5125 ;; -------------------------------------------------------------------
5126 ;; Bitfields
5127 ;; -------------------------------------------------------------------
5128
5129 (define_expand "<optab>"
5130   [(set (match_operand:DI 0 "register_operand" "=r")
5131         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5132                         (match_operand 2
5133                           "aarch64_simd_shift_imm_offset_di")
5134                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5135   ""
5136   {
5137     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5138                    1, GET_MODE_BITSIZE (DImode) - 1))
5139      FAIL;
5140   }
5141 )
5142
5143
5144 (define_insn "*<optab><mode>"
5145   [(set (match_operand:GPI 0 "register_operand" "=r")
5146         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5147                          (match_operand 2
5148                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5149                          (match_operand 3
5150                            "aarch64_simd_shift_imm_<mode>" "n")))]
5151   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5152              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5153   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5154   [(set_attr "type" "bfx")]
5155 )
5156
5157 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5158 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5159 (define_split
5160   [(set (match_operand:DI 0 "register_operand")
5161         (zero_extract:DI (match_operand:DI 1 "register_operand")
5162                          (match_operand 2
5163                            "aarch64_simd_shift_imm_offset_di")
5164                          (match_operand 3
5165                            "aarch64_simd_shift_imm_di")))]
5166   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5167              GET_MODE_BITSIZE (DImode) - 1)
5168    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5169        == GET_MODE_BITSIZE (SImode)"
5170   [(set (match_dup 0)
5171         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5172   {
5173     operands[4] = gen_lowpart (SImode, operands[1]);
5174   }
5175 )
5176
5177 ;; Bitfield Insert (insv)
5178 (define_expand "insv<mode>"
5179   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5180                           (match_operand 1 "const_int_operand")
5181                           (match_operand 2 "const_int_operand"))
5182         (match_operand:GPI 3 "general_operand"))]
5183   ""
5184 {
5185   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5186   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5187   rtx value = operands[3];
5188
5189   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5190     FAIL;
5191
5192   if (CONST_INT_P (value))
5193     {
5194       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5195
5196       /* Prefer AND/OR for inserting all zeros or all ones.  */
5197       if ((UINTVAL (value) & mask) == 0
5198            || (UINTVAL (value) & mask) == mask)
5199         FAIL;
5200
5201       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5202       if (width == 16 && (pos % 16) == 0)
5203         DONE;
5204     }
5205   operands[3] = force_reg (<MODE>mode, value);
5206 })
5207
5208 (define_insn "*insv_reg<mode>"
5209   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5210                           (match_operand 1 "const_int_operand" "n")
5211                           (match_operand 2 "const_int_operand" "n"))
5212         (match_operand:GPI 3 "register_operand" "r"))]
5213   "!(UINTVAL (operands[1]) == 0
5214      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5215          > GET_MODE_BITSIZE (<MODE>mode)))"
5216   "bfi\\t%<w>0, %<w>3, %2, %1"
5217   [(set_attr "type" "bfm")]
5218 )
5219
5220 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5221   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5222                           (match_operand 1 "const_int_operand" "n")
5223                           (match_operand 2 "const_int_operand" "n"))
5224         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5225   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5226   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5227   [(set_attr "type" "bfm")]
5228 )
5229
5230 (define_insn "*extr_insv_lower_reg<mode>"
5231   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5232                           (match_operand 1 "const_int_operand" "n")
5233                           (const_int 0))
5234         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5235                           (match_dup 1)
5236                           (match_operand 3 "const_int_operand" "n")))]
5237   "!(UINTVAL (operands[1]) == 0
5238      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5239          > GET_MODE_BITSIZE (<MODE>mode)))"
5240   "bfxil\\t%<w>0, %<w>2, %3, %1"
5241   [(set_attr "type" "bfm")]
5242 )
5243
5244 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5245   [(set (match_operand:GPI 0 "register_operand" "=r")
5246         (ashift:GPI (ANY_EXTEND:GPI
5247                      (match_operand:ALLX 1 "register_operand" "r"))
5248                     (match_operand 2 "const_int_operand" "n")))]
5249   "UINTVAL (operands[2]) < <GPI:sizen>"
5250 {
5251   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5252               ? GEN_INT (<ALLX:sizen>)
5253               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5254   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5255 }
5256   [(set_attr "type" "bfx")]
5257 )
5258
5259 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5260
5261 (define_insn "*andim_ashift<mode>_bfiz"
5262   [(set (match_operand:GPI 0 "register_operand" "=r")
5263         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5264                              (match_operand 2 "const_int_operand" "n"))
5265                  (match_operand 3 "const_int_operand" "n")))]
5266   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5267   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5268   [(set_attr "type" "bfx")]
5269 )
5270
5271 ;; Match sbfiz pattern in a shift left + shift right operation.
5272
5273 (define_insn "*ashift<mode>_extv_bfiz"
5274   [(set (match_operand:GPI 0 "register_operand" "=r")
5275         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5276                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5277                                       (const_int 0))
5278                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5279   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5280              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5281   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5282   [(set_attr "type" "bfx")]
5283 )
5284
5285 ;; When the bit position and width of the equivalent extraction add up to 32
5286 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5287 ;; zero-extension of the X-reg.
5288 (define_split
5289   [(set (match_operand:DI 0 "register_operand")
5290         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5291                              (match_operand 2 "const_int_operand"))
5292                  (match_operand 3 "const_int_operand")))]
5293  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5294   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5295       == GET_MODE_BITSIZE (SImode)"
5296   [(set (match_dup 0)
5297         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5298   {
5299     operands[4] = gen_lowpart (SImode, operands[1]);
5300   }
5301 )
5302
5303 (define_insn "bswap<mode>2"
5304   [(set (match_operand:GPI 0 "register_operand" "=r")
5305         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5306   ""
5307   "rev\\t%<w>0, %<w>1"
5308   [(set_attr "type" "rev")]
5309 )
5310
5311 (define_insn "bswaphi2"
5312   [(set (match_operand:HI 0 "register_operand" "=r")
5313         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5314   ""
5315   "rev16\\t%w0, %w1"
5316   [(set_attr "type" "rev")]
5317 )
5318
5319 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5320 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5321 ;; each valid permutation.
5322
5323 (define_insn "rev16<mode>2"
5324   [(set (match_operand:GPI 0 "register_operand" "=r")
5325         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5326                                       (const_int 8))
5327                           (match_operand:GPI 3 "const_int_operand" "n"))
5328                  (and:GPI (lshiftrt:GPI (match_dup 1)
5329                                         (const_int 8))
5330                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5331   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5332    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5333   "rev16\\t%<w>0, %<w>1"
5334   [(set_attr "type" "rev")]
5335 )
5336
5337 (define_insn "rev16<mode>2_alt"
5338   [(set (match_operand:GPI 0 "register_operand" "=r")
5339         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5340                                         (const_int 8))
5341                           (match_operand:GPI 2 "const_int_operand" "n"))
5342                  (and:GPI (ashift:GPI (match_dup 1)
5343                                       (const_int 8))
5344                           (match_operand:GPI 3 "const_int_operand" "n"))))]
5345   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5346    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5347   "rev16\\t%<w>0, %<w>1"
5348   [(set_attr "type" "rev")]
5349 )
5350
5351 ;; zero_extend version of above
5352 (define_insn "*bswapsi2_uxtw"
5353   [(set (match_operand:DI 0 "register_operand" "=r")
5354         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5355   ""
5356   "rev\\t%w0, %w1"
5357   [(set_attr "type" "rev")]
5358 )
5359
5360 ;; -------------------------------------------------------------------
5361 ;; Floating-point intrinsics
5362 ;; -------------------------------------------------------------------
5363
5364 ;; frint floating-point round to integral standard patterns.
5365 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5366
5367 (define_insn "<frint_pattern><mode>2"
5368   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5369         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5370          FRINT))]
5371   "TARGET_FLOAT"
5372   "frint<frint_suffix>\\t%<s>0, %<s>1"
5373   [(set_attr "type" "f_rint<stype>")]
5374 )
5375
5376 ;; frcvt floating-point round to integer and convert standard patterns.
5377 ;; Expands to lbtrunc, lceil, lfloor, lround.
5378 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5379   [(set (match_operand:GPI 0 "register_operand" "=r")
5380         (FIXUORS:GPI
5381           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5382            FCVT)))]
5383   "TARGET_FLOAT"
5384   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5385   [(set_attr "type" "f_cvtf2i")]
5386 )
5387
5388 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5389   [(set (match_operand:GPI 0 "register_operand" "=r")
5390         (FIXUORS:GPI
5391           (mult:GPF
5392             (match_operand:GPF 1 "register_operand" "w")
5393             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5394   "TARGET_FLOAT
5395    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5396                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5397   {
5398     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5399     char buf[64];
5400     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5401     output_asm_insn (buf, operands);
5402     return "";
5403   }
5404   [(set_attr "type" "f_cvtf2i")]
5405 )
5406
5407 ;; fma - expand fma into patterns with the accumulator operand first since
5408 ;; reusing the accumulator results in better register allocation.
5409 ;; The register allocator considers copy preferences in operand order,
5410 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5411
5412 (define_expand "fma<mode>4"
5413   [(set (match_operand:GPF_F16 0 "register_operand")
5414         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5415                      (match_operand:GPF_F16 2 "register_operand")
5416                      (match_operand:GPF_F16 3 "register_operand")))]
5417   "TARGET_FLOAT"
5418 )
5419
5420 (define_insn "*aarch64_fma<mode>4"
5421   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5422         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5423                      (match_operand:GPF_F16 3 "register_operand" "w")
5424                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5425   "TARGET_FLOAT"
5426   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5427   [(set_attr "type" "fmac<stype>")]
5428 )
5429
5430 (define_expand "fnma<mode>4"
5431   [(set (match_operand:GPF_F16 0 "register_operand")
5432         (fma:GPF_F16
5433           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5434           (match_operand:GPF_F16 2 "register_operand")
5435           (match_operand:GPF_F16 3 "register_operand")))]
5436   "TARGET_FLOAT"
5437 )
5438
5439 (define_insn "*aarch64_fnma<mode>4"
5440   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5441         (fma:GPF_F16
5442           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5443           (match_operand:GPF_F16 3 "register_operand" "w")
5444           (match_operand:GPF_F16 1 "register_operand" "w")))]
5445   "TARGET_FLOAT"
5446   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5447   [(set_attr "type" "fmac<stype>")]
5448 )
5449
5450
5451 (define_expand "fms<mode>4"
5452   [(set (match_operand:GPF 0 "register_operand")
5453         (fma:GPF (match_operand:GPF 1 "register_operand")
5454                  (match_operand:GPF 2 "register_operand")
5455                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5456   "TARGET_FLOAT"
5457 )
5458
5459 (define_insn "*aarch64_fms<mode>4"
5460   [(set (match_operand:GPF 0 "register_operand" "=w")
5461         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5462                  (match_operand:GPF 3 "register_operand" "w")
5463                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5464   "TARGET_FLOAT"
5465   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5466   [(set_attr "type" "fmac<s>")]
5467 )
5468
5469 (define_expand "fnms<mode>4"
5470   [(set (match_operand:GPF 0 "register_operand")
5471         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5472                  (match_operand:GPF 2 "register_operand")
5473                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5474   "TARGET_FLOAT"
5475 )
5476
5477 (define_insn "*aarch64_fnms<mode>4"
5478   [(set (match_operand:GPF 0 "register_operand" "=w")
5479         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5480                  (match_operand:GPF 3 "register_operand" "w")
5481                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5482   "TARGET_FLOAT"
5483   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5484   [(set_attr "type" "fmac<s>")]
5485 )
5486
5487 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5488 (define_insn "*aarch64_fnmadd<mode>4"
5489   [(set (match_operand:GPF 0 "register_operand" "=w")
5490         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5491                           (match_operand:GPF 3 "register_operand" "w")
5492                           (match_operand:GPF 1 "register_operand" "w"))))]
5493   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5494   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5495   [(set_attr "type" "fmac<s>")]
5496 )
5497
5498 ;; -------------------------------------------------------------------
5499 ;; Floating-point conversions
5500 ;; -------------------------------------------------------------------
5501
5502 (define_insn "extendsfdf2"
5503   [(set (match_operand:DF 0 "register_operand" "=w")
5504         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5505   "TARGET_FLOAT"
5506   "fcvt\\t%d0, %s1"
5507   [(set_attr "type" "f_cvt")]
5508 )
5509
5510 (define_insn "extendhfsf2"
5511   [(set (match_operand:SF 0 "register_operand" "=w")
5512         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5513   "TARGET_FLOAT"
5514   "fcvt\\t%s0, %h1"
5515   [(set_attr "type" "f_cvt")]
5516 )
5517
5518 (define_insn "extendhfdf2"
5519   [(set (match_operand:DF 0 "register_operand" "=w")
5520         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5521   "TARGET_FLOAT"
5522   "fcvt\\t%d0, %h1"
5523   [(set_attr "type" "f_cvt")]
5524 )
5525
5526 (define_insn "truncdfsf2"
5527   [(set (match_operand:SF 0 "register_operand" "=w")
5528         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5529   "TARGET_FLOAT"
5530   "fcvt\\t%s0, %d1"
5531   [(set_attr "type" "f_cvt")]
5532 )
5533
5534 (define_insn "truncsfhf2"
5535   [(set (match_operand:HF 0 "register_operand" "=w")
5536         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5537   "TARGET_FLOAT"
5538   "fcvt\\t%h0, %s1"
5539   [(set_attr "type" "f_cvt")]
5540 )
5541
5542 (define_insn "truncdfhf2"
5543   [(set (match_operand:HF 0 "register_operand" "=w")
5544         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5545   "TARGET_FLOAT"
5546   "fcvt\\t%h0, %d1"
5547   [(set_attr "type" "f_cvt")]
5548 )
5549
5550 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5551 ;; and making r = w more expensive
5552
5553 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5554   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5555         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5556   "TARGET_FLOAT"
5557   "@
5558    fcvtz<su>\t%<s>0, %<s>1
5559    fcvtz<su>\t%<w>0, %<s>1"
5560   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5561 )
5562
5563 ;; Convert HF -> SI or DI
5564
5565 (define_insn "<optab>_trunchf<GPI:mode>2"
5566   [(set (match_operand:GPI 0 "register_operand" "=r")
5567         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5568   "TARGET_FP_F16INST"
5569   "fcvtz<su>\t%<w>0, %h1"
5570   [(set_attr "type" "f_cvtf2i")]
5571 )
5572
5573 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5574 ;; input in a fp register and output in a integer register
5575
5576 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5577   [(set (match_operand:GPI 0 "register_operand" "=r")
5578         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5579   "TARGET_FLOAT"
5580   "fcvtz<su>\t%<w>0, %<fpw>1"
5581   [(set_attr "type" "f_cvtf2i")]
5582 )
5583
5584 (define_insn "*fix_to_zero_extend<mode>di2"
5585   [(set (match_operand:DI 0 "register_operand" "=r")
5586         (zero_extend:DI
5587          (unsigned_fix:SI
5588           (match_operand:GPF 1 "register_operand" "w"))))]
5589   "TARGET_FLOAT"
5590   "fcvtzu\t%w0, %<s>1"
5591   [(set_attr "type" "f_cvtf2i")]
5592 )
5593
5594 (define_insn "<optab><fcvt_target><GPF:mode>2"
5595   [(set (match_operand:GPF 0 "register_operand" "=w,w")
5596         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
5597   "TARGET_FLOAT"
5598   "@
5599    <su_optab>cvtf\t%<GPF:s>0, %<s>1
5600    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5601   [(set_attr "simd" "yes,no")
5602    (set_attr "fp" "no,yes")
5603    (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
5604 )
5605
5606 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5607   [(set (match_operand:GPF 0 "register_operand" "=w")
5608         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5609   "TARGET_FLOAT"
5610   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5611   [(set_attr "type" "f_cvti2f")]
5612 )
5613
5614 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5615 ;; midend will arrange for an SImode conversion to HFmode to first go
5616 ;; through DFmode, then to HFmode.  But first it will try converting
5617 ;; to DImode then down, which would match our DImode pattern below and
5618 ;; give very poor code-generation.  So, we must provide our own emulation
5619 ;; of the mid-end logic.
5620
5621 (define_insn "aarch64_fp16_<optab><mode>hf2"
5622   [(set (match_operand:HF 0 "register_operand" "=w")
5623         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5624   "TARGET_FP_F16INST"
5625   "<su_optab>cvtf\t%h0, %<w>1"
5626   [(set_attr "type" "f_cvti2f")]
5627 )
5628
5629 (define_expand "<optab>sihf2"
5630   [(set (match_operand:HF 0 "register_operand")
5631         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5632   "TARGET_FLOAT"
5633 {
5634   if (TARGET_FP_F16INST)
5635     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5636   else
5637     {
5638       rtx convert_target = gen_reg_rtx (DFmode);
5639       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5640       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5641     }
5642   DONE;
5643 }
5644 )
5645
5646 ;; For DImode there is no wide enough floating-point mode that we
5647 ;; can convert through natively (TFmode would work, but requires a library
5648 ;; call).  However, we know that any value >= 65504 will be rounded
5649 ;; to infinity on conversion.  This is well within the range of SImode, so
5650 ;; we can:
5651 ;;   Saturate to SImode.
5652 ;;   Convert from that to DFmode
5653 ;;   Convert from that to HFmode (phew!).
5654 ;; Note that the saturation to SImode requires the SIMD extensions.  If
5655 ;; we ever need to provide this pattern where the SIMD extensions are not
5656 ;; available, we would need a different approach.
5657
5658 (define_expand "<optab>dihf2"
5659   [(set (match_operand:HF 0 "register_operand")
5660         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5661   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5662 {
5663   if (TARGET_FP_F16INST)
5664     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5665   else
5666     {
5667       rtx sat_target = gen_reg_rtx (SImode);
5668       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5669       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5670     }
5671
5672   DONE;
5673 }
5674 )
5675
5676 ;; Convert between fixed-point and floating-point (scalar modes)
5677
5678 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
5679   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
5680         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
5681                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5682          FCVT_F2FIXED))]
5683   ""
5684   "@
5685    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
5686    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
5687   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
5688    (set_attr "fp" "yes, *")
5689    (set_attr "simd" "*, yes")]
5690 )
5691
5692 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
5693   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
5694         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
5695                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5696          FCVT_FIXED2F))]
5697   ""
5698   "@
5699    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
5700    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
5701   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
5702    (set_attr "fp" "yes, *")
5703    (set_attr "simd" "*, yes")]
5704 )
5705
5706 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
5707   [(set (match_operand:GPI 0 "register_operand" "=r")
5708         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
5709                      (match_operand:SI 2 "immediate_operand" "i")]
5710          FCVT_F2FIXED))]
5711   "TARGET_FP_F16INST"
5712    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
5713   [(set_attr "type" "f_cvtf2i")]
5714 )
5715
5716 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
5717   [(set (match_operand:HF 0 "register_operand" "=w")
5718         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
5719                     (match_operand:SI 2 "immediate_operand" "i")]
5720          FCVT_FIXED2F))]
5721   "TARGET_FP_F16INST"
5722   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
5723   [(set_attr "type" "f_cvti2f")]
5724 )
5725
5726 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
5727   [(set (match_operand:HI 0 "register_operand" "=w")
5728         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
5729                     (match_operand:SI 2 "immediate_operand" "i")]
5730          FCVT_F2FIXED))]
5731   "TARGET_SIMD"
5732   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
5733   [(set_attr "type" "neon_fp_to_int_s")]
5734 )
5735
5736 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
5737   [(set (match_operand:HF 0 "register_operand" "=w")
5738         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
5739                     (match_operand:SI 2 "immediate_operand" "i")]
5740          FCVT_FIXED2F))]
5741   "TARGET_SIMD"
5742   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
5743   [(set_attr "type" "neon_int_to_fp_s")]
5744 )
5745
5746 ;; -------------------------------------------------------------------
5747 ;; Floating-point arithmetic
5748 ;; -------------------------------------------------------------------
5749
5750 (define_insn "add<mode>3"
5751   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5752         (plus:GPF_F16
5753          (match_operand:GPF_F16 1 "register_operand" "w")
5754          (match_operand:GPF_F16 2 "register_operand" "w")))]
5755   "TARGET_FLOAT"
5756   "fadd\\t%<s>0, %<s>1, %<s>2"
5757   [(set_attr "type" "fadd<stype>")]
5758 )
5759
5760 (define_insn "sub<mode>3"
5761   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5762         (minus:GPF_F16
5763          (match_operand:GPF_F16 1 "register_operand" "w")
5764          (match_operand:GPF_F16 2 "register_operand" "w")))]
5765   "TARGET_FLOAT"
5766   "fsub\\t%<s>0, %<s>1, %<s>2"
5767   [(set_attr "type" "fadd<stype>")]
5768 )
5769
5770 (define_insn "mul<mode>3"
5771   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5772         (mult:GPF_F16
5773          (match_operand:GPF_F16 1 "register_operand" "w")
5774          (match_operand:GPF_F16 2 "register_operand" "w")))]
5775   "TARGET_FLOAT"
5776   "fmul\\t%<s>0, %<s>1, %<s>2"
5777   [(set_attr "type" "fmul<stype>")]
5778 )
5779
5780 (define_insn "*fnmul<mode>3"
5781   [(set (match_operand:GPF 0 "register_operand" "=w")
5782         (mult:GPF
5783                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
5784                  (match_operand:GPF 2 "register_operand" "w")))]
5785   "TARGET_FLOAT && !flag_rounding_math"
5786   "fnmul\\t%<s>0, %<s>1, %<s>2"
5787   [(set_attr "type" "fmul<s>")]
5788 )
5789
5790 (define_insn "*fnmul<mode>3"
5791   [(set (match_operand:GPF 0 "register_operand" "=w")
5792         (neg:GPF (mult:GPF
5793                  (match_operand:GPF 1 "register_operand" "w")
5794                  (match_operand:GPF 2 "register_operand" "w"))))]
5795   "TARGET_FLOAT"
5796   "fnmul\\t%<s>0, %<s>1, %<s>2"
5797   [(set_attr "type" "fmul<s>")]
5798 )
5799
5800 (define_expand "div<mode>3"
5801  [(set (match_operand:GPF_F16 0 "register_operand")
5802        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
5803                     (match_operand:GPF_F16 2 "register_operand")))]
5804  "TARGET_FLOAT"
5805 {
5806   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
5807     DONE;
5808
5809   operands[1] = force_reg (<MODE>mode, operands[1]);
5810 })
5811
5812 (define_insn "*div<mode>3"
5813   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5814         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
5815                      (match_operand:GPF_F16 2 "register_operand" "w")))]
5816   "TARGET_FLOAT"
5817   "fdiv\\t%<s>0, %<s>1, %<s>2"
5818   [(set_attr "type" "fdiv<stype>")]
5819 )
5820
5821 (define_insn "neg<mode>2"
5822   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5823         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5824   "TARGET_FLOAT"
5825   "fneg\\t%<s>0, %<s>1"
5826   [(set_attr "type" "ffarith<stype>")]
5827 )
5828
5829 (define_expand "sqrt<mode>2"
5830   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5831         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5832   "TARGET_FLOAT"
5833 {
5834   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
5835     DONE;
5836 })
5837
5838 (define_insn "*sqrt<mode>2"
5839   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5840         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5841   "TARGET_FLOAT"
5842   "fsqrt\\t%<s>0, %<s>1"
5843   [(set_attr "type" "fsqrt<stype>")]
5844 )
5845
5846 (define_insn "abs<mode>2"
5847   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5848         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5849   "TARGET_FLOAT"
5850   "fabs\\t%<s>0, %<s>1"
5851   [(set_attr "type" "ffarith<stype>")]
5852 )
5853
5854 ;; Given that smax/smin do not specify the result when either input is NaN,
5855 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
5856 ;; for smin.
5857
5858 (define_insn "smax<mode>3"
5859   [(set (match_operand:GPF 0 "register_operand" "=w")
5860         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
5861                   (match_operand:GPF 2 "register_operand" "w")))]
5862   "TARGET_FLOAT"
5863   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
5864   [(set_attr "type" "f_minmax<s>")]
5865 )
5866
5867 (define_insn "smin<mode>3"
5868   [(set (match_operand:GPF 0 "register_operand" "=w")
5869         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
5870                   (match_operand:GPF 2 "register_operand" "w")))]
5871   "TARGET_FLOAT"
5872   "fminnm\\t%<s>0, %<s>1, %<s>2"
5873   [(set_attr "type" "f_minmax<s>")]
5874 )
5875
5876 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
5877 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
5878 ;; which implement the IEEE fmax ()/fmin () functions.
5879 (define_insn "<maxmin_uns><mode>3"
5880   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5881         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
5882                      (match_operand:GPF_F16 2 "register_operand" "w")]
5883                      FMAXMIN_UNS))]
5884   "TARGET_FLOAT"
5885   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
5886   [(set_attr "type" "f_minmax<stype>")]
5887 )
5888
5889 (define_expand "lrint<GPF:mode><GPI:mode>2"
5890   [(match_operand:GPI 0 "register_operand")
5891    (match_operand:GPF 1 "register_operand")]
5892   "TARGET_FLOAT
5893    && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
5894    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
5895 {
5896   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
5897   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
5898   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
5899   DONE;
5900 }
5901 )
5902
5903 ;; For copysign (x, y), we want to generate:
5904 ;;
5905 ;;   LDR d2, #(1 << 63)
5906 ;;   BSL v2.8b, [y], [x]
5907 ;;
5908 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
5909 ;; aarch64_simd_bsldf will select the best suited of these instructions
5910 ;; to generate based on register allocation, and knows how to partially
5911 ;; constant fold based on the values of X and Y, so expand through that.
5912
5913 (define_expand "copysigndf3"
5914   [(match_operand:DF 0 "register_operand")
5915    (match_operand:DF 1 "register_operand")
5916    (match_operand:DF 2 "register_operand")]
5917   "TARGET_FLOAT && TARGET_SIMD"
5918 {
5919   rtx mask = gen_reg_rtx (DImode);
5920   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
5921   emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
5922                                      operands[2], operands[1]));
5923   DONE;
5924 }
5925 )
5926
5927 ;; As above, but we must first get to a 64-bit value if we wish to use
5928 ;; aarch64_simd_bslv2sf.
5929
5930 (define_expand "copysignsf3"
5931   [(match_operand:SF 0 "register_operand")
5932    (match_operand:SF 1 "register_operand")
5933    (match_operand:SF 2 "register_operand")]
5934   "TARGET_FLOAT && TARGET_SIMD"
5935 {
5936   rtx v_bitmask = gen_reg_rtx (V2SImode);
5937
5938   /* Juggle modes to get us in to a vector mode for BSL.  */
5939   rtx op1 = lowpart_subreg (DImode, operands[1], SFmode);
5940   rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
5941   rtx tmp = gen_reg_rtx (V2SFmode);
5942   emit_move_insn (v_bitmask,
5943                   aarch64_simd_gen_const_vector_dup (V2SImode,
5944                                                      HOST_WIDE_INT_M1U << 31));
5945   emit_insn (gen_aarch64_simd_bslv2sf (tmp, v_bitmask, op2, op1));
5946   emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
5947   DONE;
5948 }
5949 )
5950
5951 ;; For xorsign (x, y), we want to generate:
5952 ;;
5953 ;; LDR   d2, #1<<63
5954 ;; AND   v3.8B, v1.8B, v2.8B
5955 ;; EOR   v0.8B, v0.8B, v3.8B
5956 ;;
5957
5958 (define_expand "xorsign<mode>3"
5959   [(match_operand:GPF 0 "register_operand")
5960    (match_operand:GPF 1 "register_operand")
5961    (match_operand:GPF 2 "register_operand")]
5962   "TARGET_FLOAT && TARGET_SIMD"
5963 {
5964
5965   machine_mode imode = <V_INT_EQUIV>mode;
5966   rtx mask = gen_reg_rtx (imode);
5967   rtx op1x = gen_reg_rtx (imode);
5968   rtx op2x = gen_reg_rtx (imode);
5969
5970   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
5971   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
5972                                                      imode)));
5973
5974   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
5975                                     lowpart_subreg (imode, operands[2],
5976                                                     <MODE>mode)));
5977   emit_insn (gen_xor<v_int_equiv>3 (op1x,
5978                                     lowpart_subreg (imode, operands[1],
5979                                                     <MODE>mode),
5980                                     op2x));
5981   emit_move_insn (operands[0],
5982                   lowpart_subreg (<MODE>mode, op1x, imode));
5983   DONE;
5984 }
5985 )
5986
5987 ;; -------------------------------------------------------------------
5988 ;; Reload support
5989 ;; -------------------------------------------------------------------
5990 ;; Reload Scalar Floating point modes from constant pool.
5991 ;; The AArch64 port doesn't have __int128 constant move support.
5992 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
5993  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
5994        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
5995   (clobber (match_operand:P 2 "register_operand" "=&r"))]
5996  "TARGET_FLOAT"
5997  {
5998    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5999    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6000    DONE;
6001  }
6002 )
6003
6004 ;; Reload Vector modes from constant pool.
6005 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
6006  [(set (match_operand:VALL 0 "register_operand" "=w")
6007        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6008   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6009  "TARGET_FLOAT"
6010  {
6011    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6012    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6013    DONE;
6014  }
6015 )
6016
6017 (define_expand "aarch64_reload_mov<mode>"
6018   [(set (match_operand:TX 0 "register_operand" "=w")
6019         (match_operand:TX 1 "register_operand" "w"))
6020    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6021   ]
6022   "TARGET_FLOAT"
6023   {
6024     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6025     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6026     gen_aarch64_movtilow_tilow (op0, op1);
6027     gen_aarch64_movdi_tihigh (operands[2], op1);
6028     gen_aarch64_movtihigh_di (op0, operands[2]);
6029     DONE;
6030   }
6031 )
6032
6033 ;; The following secondary reload helpers patterns are invoked
6034 ;; after or during reload as we don't want these patterns to start
6035 ;; kicking in during the combiner.
6036
6037 (define_insn "aarch64_movdi_<mode>low"
6038   [(set (match_operand:DI 0 "register_operand" "=r")
6039         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6040                          (const_int 64) (const_int 0)))]
6041   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6042   "fmov\\t%x0, %d1"
6043   [(set_attr "type" "f_mrc")
6044    (set_attr "length" "4")
6045   ])
6046
6047 (define_insn "aarch64_movdi_<mode>high"
6048   [(set (match_operand:DI 0 "register_operand" "=r")
6049         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6050                          (const_int 64) (const_int 64)))]
6051   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6052   "fmov\\t%x0, %1.d[1]"
6053   [(set_attr "type" "f_mrc")
6054    (set_attr "length" "4")
6055   ])
6056
6057 (define_insn "aarch64_mov<mode>high_di"
6058   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6059                          (const_int 64) (const_int 64))
6060         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6061   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6062   "fmov\\t%0.d[1], %x1"
6063   [(set_attr "type" "f_mcr")
6064    (set_attr "length" "4")
6065   ])
6066
6067 (define_insn "aarch64_mov<mode>low_di"
6068   [(set (match_operand:TX 0 "register_operand" "=w")
6069         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6070   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6071   "fmov\\t%d0, %x1"
6072   [(set_attr "type" "f_mcr")
6073    (set_attr "length" "4")
6074   ])
6075
6076 (define_insn "aarch64_movtilow_tilow"
6077   [(set (match_operand:TI 0 "register_operand" "=w")
6078         (zero_extend:TI
6079           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6080   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6081   "fmov\\t%d0, %d1"
6082   [(set_attr "type" "fmov")
6083    (set_attr "length" "4")
6084   ])
6085
6086 ;; There is a deliberate reason why the parameters of high and lo_sum's
6087 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6088 ;; and lo_sum's to be used with the labels defining the jump tables in
6089 ;; rodata section.
6090
6091 (define_expand "add_losym"
6092   [(set (match_operand 0 "register_operand" "=r")
6093         (lo_sum (match_operand 1 "register_operand" "r")
6094                 (match_operand 2 "aarch64_valid_symref" "S")))]
6095   ""
6096 {
6097   machine_mode mode = GET_MODE (operands[0]);
6098
6099   emit_insn ((mode == DImode
6100               ? gen_add_losym_di
6101               : gen_add_losym_si) (operands[0],
6102                                    operands[1],
6103                                    operands[2]));
6104   DONE;
6105 })
6106
6107 (define_insn "add_losym_<mode>"
6108   [(set (match_operand:P 0 "register_operand" "=r")
6109         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6110                   (match_operand 2 "aarch64_valid_symref" "S")))]
6111   ""
6112   "add\\t%<w>0, %<w>1, :lo12:%c2"
6113   [(set_attr "type" "alu_imm")]
6114 )
6115
6116 (define_insn "ldr_got_small_<mode>"
6117   [(set (match_operand:PTR 0 "register_operand" "=r")
6118         (unspec:PTR [(mem:PTR (lo_sum:PTR
6119                               (match_operand:PTR 1 "register_operand" "r")
6120                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6121                     UNSPEC_GOTSMALLPIC))]
6122   ""
6123   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6124   [(set_attr "type" "load_<ldst_sz>")]
6125 )
6126
6127 (define_insn "ldr_got_small_sidi"
6128   [(set (match_operand:DI 0 "register_operand" "=r")
6129         (zero_extend:DI
6130          (unspec:SI [(mem:SI (lo_sum:DI
6131                              (match_operand:DI 1 "register_operand" "r")
6132                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6133                     UNSPEC_GOTSMALLPIC)))]
6134   "TARGET_ILP32"
6135   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6136   [(set_attr "type" "load_4")]
6137 )
6138
6139 (define_insn "ldr_got_small_28k_<mode>"
6140   [(set (match_operand:PTR 0 "register_operand" "=r")
6141         (unspec:PTR [(mem:PTR (lo_sum:PTR
6142                               (match_operand:PTR 1 "register_operand" "r")
6143                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6144                     UNSPEC_GOTSMALLPIC28K))]
6145   ""
6146   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6147   [(set_attr "type" "load_<ldst_sz>")]
6148 )
6149
6150 (define_insn "ldr_got_small_28k_sidi"
6151   [(set (match_operand:DI 0 "register_operand" "=r")
6152         (zero_extend:DI
6153          (unspec:SI [(mem:SI (lo_sum:DI
6154                              (match_operand:DI 1 "register_operand" "r")
6155                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6156                     UNSPEC_GOTSMALLPIC28K)))]
6157   "TARGET_ILP32"
6158   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6159   [(set_attr "type" "load_4")]
6160 )
6161
6162 (define_insn "ldr_got_tiny"
6163   [(set (match_operand:DI 0 "register_operand" "=r")
6164         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6165                    UNSPEC_GOTTINYPIC))]
6166   ""
6167   "ldr\\t%0, %L1"
6168   [(set_attr "type" "load_8")]
6169 )
6170
6171 (define_insn "aarch64_load_tp_hard"
6172   [(set (match_operand:DI 0 "register_operand" "=r")
6173         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6174   ""
6175   "mrs\\t%0, tpidr_el0"
6176   [(set_attr "type" "mrs")]
6177 )
6178
6179 ;; The TLS ABI specifically requires that the compiler does not schedule
6180 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6181 ;; Therefore we treat the stubs as an atomic sequence.
6182 (define_expand "tlsgd_small_<mode>"
6183  [(parallel [(set (match_operand 0 "register_operand" "")
6184                   (call (mem:DI (match_dup 2)) (const_int 1)))
6185              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
6186              (clobber (reg:DI LR_REGNUM))])]
6187  ""
6188 {
6189   operands[2] = aarch64_tls_get_addr ();
6190 })
6191
6192 (define_insn "*tlsgd_small_<mode>"
6193   [(set (match_operand 0 "register_operand" "")
6194         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6195    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6196    (clobber (reg:DI LR_REGNUM))
6197   ]
6198   ""
6199   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6200   [(set_attr "type" "call")
6201    (set_attr "length" "16")])
6202
6203 (define_insn "tlsie_small_<mode>"
6204   [(set (match_operand:PTR 0 "register_operand" "=r")
6205         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6206                    UNSPEC_GOTSMALLTLS))]
6207   ""
6208   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6209   [(set_attr "type" "load_4")
6210    (set_attr "length" "8")]
6211 )
6212
6213 (define_insn "tlsie_small_sidi"
6214   [(set (match_operand:DI 0 "register_operand" "=r")
6215         (zero_extend:DI
6216           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6217                       UNSPEC_GOTSMALLTLS)))]
6218   ""
6219   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6220   [(set_attr "type" "load_4")
6221    (set_attr "length" "8")]
6222 )
6223
6224 (define_insn "tlsie_tiny_<mode>"
6225   [(set (match_operand:PTR 0 "register_operand" "=&r")
6226         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6227                      (match_operand:PTR 2 "register_operand" "r")]
6228                    UNSPEC_GOTTINYTLS))]
6229   ""
6230   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6231   [(set_attr "type" "multiple")
6232    (set_attr "length" "8")]
6233 )
6234
6235 (define_insn "tlsie_tiny_sidi"
6236   [(set (match_operand:DI 0 "register_operand" "=&r")
6237         (zero_extend:DI
6238           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6239                       (match_operand:DI 2 "register_operand" "r")
6240                       ]
6241                       UNSPEC_GOTTINYTLS)))]
6242   ""
6243   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6244   [(set_attr "type" "multiple")
6245    (set_attr "length" "8")]
6246 )
6247
6248 (define_insn "tlsle12_<mode>"
6249   [(set (match_operand:P 0 "register_operand" "=r")
6250         (unspec:P [(match_operand:P 1 "register_operand" "r")
6251                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6252                    UNSPEC_TLSLE12))]
6253   ""
6254   "add\\t%<w>0, %<w>1, #%L2";
6255   [(set_attr "type" "alu_sreg")
6256    (set_attr "length" "4")]
6257 )
6258
6259 (define_insn "tlsle24_<mode>"
6260   [(set (match_operand:P 0 "register_operand" "=r")
6261         (unspec:P [(match_operand:P 1 "register_operand" "r")
6262                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6263                    UNSPEC_TLSLE24))]
6264   ""
6265   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6266   [(set_attr "type" "multiple")
6267    (set_attr "length" "8")]
6268 )
6269
6270 (define_insn "tlsle32_<mode>"
6271   [(set (match_operand:P 0 "register_operand" "=r")
6272         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6273                    UNSPEC_TLSLE32))]
6274   ""
6275   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6276   [(set_attr "type" "multiple")
6277    (set_attr "length" "8")]
6278 )
6279
6280 (define_insn "tlsle48_<mode>"
6281   [(set (match_operand:P 0 "register_operand" "=r")
6282         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6283                    UNSPEC_TLSLE48))]
6284   ""
6285   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6286   [(set_attr "type" "multiple")
6287    (set_attr "length" "12")]
6288 )
6289
6290 (define_expand "tlsdesc_small_<mode>"
6291   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6292   "TARGET_TLS_DESC"
6293   {
6294     if (TARGET_SVE)
6295       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6296     else
6297       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6298     DONE;
6299   }
6300 )
6301
6302 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6303 ;; R0 and LR.
6304 (define_insn "tlsdesc_small_advsimd_<mode>"
6305   [(set (reg:PTR R0_REGNUM)
6306         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6307                     UNSPEC_TLSDESC))
6308    (clobber (reg:DI LR_REGNUM))
6309    (clobber (reg:CC CC_REGNUM))
6310    (clobber (match_scratch:DI 1 "=r"))]
6311   "TARGET_TLS_DESC && !TARGET_SVE"
6312   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6313   [(set_attr "type" "call")
6314    (set_attr "length" "16")])
6315
6316 ;; For SVE, model tlsdesc calls as clobbering all vector and predicate
6317 ;; registers, on top of the usual R0 and LR.  In reality the calls
6318 ;; preserve the low 128 bits of the vector registers, but we don't
6319 ;; yet have a way of representing that in the instruction pattern.
6320 (define_insn "tlsdesc_small_sve_<mode>"
6321   [(set (reg:PTR R0_REGNUM)
6322         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6323                     UNSPEC_TLSDESC))
6324    (clobber (reg:DI LR_REGNUM))
6325    (clobber (reg:CC CC_REGNUM))
6326    (clobber (reg:XI V0_REGNUM))
6327    (clobber (reg:XI V4_REGNUM))
6328    (clobber (reg:XI V8_REGNUM))
6329    (clobber (reg:XI V12_REGNUM))
6330    (clobber (reg:XI V16_REGNUM))
6331    (clobber (reg:XI V20_REGNUM))
6332    (clobber (reg:XI V24_REGNUM))
6333    (clobber (reg:XI V28_REGNUM))
6334    (clobber (reg:VNx2BI P0_REGNUM))
6335    (clobber (reg:VNx2BI P1_REGNUM))
6336    (clobber (reg:VNx2BI P2_REGNUM))
6337    (clobber (reg:VNx2BI P3_REGNUM))
6338    (clobber (reg:VNx2BI P4_REGNUM))
6339    (clobber (reg:VNx2BI P5_REGNUM))
6340    (clobber (reg:VNx2BI P6_REGNUM))
6341    (clobber (reg:VNx2BI P7_REGNUM))
6342    (clobber (reg:VNx2BI P8_REGNUM))
6343    (clobber (reg:VNx2BI P9_REGNUM))
6344    (clobber (reg:VNx2BI P10_REGNUM))
6345    (clobber (reg:VNx2BI P11_REGNUM))
6346    (clobber (reg:VNx2BI P12_REGNUM))
6347    (clobber (reg:VNx2BI P13_REGNUM))
6348    (clobber (reg:VNx2BI P14_REGNUM))
6349    (clobber (reg:VNx2BI P15_REGNUM))
6350    (clobber (match_scratch:DI 1 "=r"))]
6351   "TARGET_TLS_DESC && TARGET_SVE"
6352   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6353   [(set_attr "type" "call")
6354    (set_attr "length" "16")])
6355
6356 (define_insn "stack_tie"
6357   [(set (mem:BLK (scratch))
6358         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6359                      (match_operand:DI 1 "register_operand" "rk")]
6360                     UNSPEC_PRLG_STK))]
6361   ""
6362   ""
6363   [(set_attr "length" "0")]
6364 )
6365
6366 ;; Pointer authentication patterns are always provided.  In architecture
6367 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6368 ;; This lets the user write portable software which authenticates pointers
6369 ;; when run on something which implements ARMv8.3-A, and which runs
6370 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6371 ;; implemented.
6372
6373 ;; Signing/Authenticating R30 using SP as the salt.
6374
6375 (define_insn "<pauth_mnem_prefix>sp"
6376   [(set (reg:DI R30_REGNUM)
6377         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6378   ""
6379   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
6380 )
6381
6382 ;; Signing/Authenticating X17 using X16 as the salt.
6383
6384 (define_insn "<pauth_mnem_prefix>1716"
6385   [(set (reg:DI R17_REGNUM)
6386         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6387   ""
6388   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
6389 )
6390
6391 ;; Stripping the signature in R30.
6392
6393 (define_insn "xpaclri"
6394   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6395   ""
6396   "hint\t7 // xpaclri"
6397 )
6398
6399 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6400 ;; all of memory.  This blocks insns from being moved across this point.
6401
6402 (define_insn "blockage"
6403   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6404   ""
6405   ""
6406   [(set_attr "length" "0")
6407    (set_attr "type" "block")]
6408 )
6409
6410 (define_insn "probe_stack_range"
6411   [(set (match_operand:DI 0 "register_operand" "=r")
6412         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6413                              (match_operand:DI 2 "register_operand" "r")]
6414                               UNSPECV_PROBE_STACK_RANGE))]
6415   ""
6416 {
6417   return aarch64_output_probe_stack_range (operands[0], operands[2]);
6418 }
6419   [(set_attr "length" "32")]
6420 )
6421
6422 ;; Named pattern for expanding thread pointer reference.
6423 (define_expand "get_thread_pointerdi"
6424   [(match_operand:DI 0 "register_operand" "=r")]
6425   ""
6426 {
6427   rtx tmp = aarch64_load_tp (operands[0]);
6428   if (tmp != operands[0])
6429     emit_move_insn (operands[0], tmp);
6430   DONE;
6431 })
6432
6433 ;; Named patterns for stack smashing protection.
6434 (define_expand "stack_protect_set"
6435   [(match_operand 0 "memory_operand")
6436    (match_operand 1 "memory_operand")]
6437   ""
6438 {
6439   machine_mode mode = GET_MODE (operands[0]);
6440
6441   emit_insn ((mode == DImode
6442               ? gen_stack_protect_set_di
6443               : gen_stack_protect_set_si) (operands[0], operands[1]));
6444   DONE;
6445 })
6446
6447 (define_insn "stack_protect_set_<mode>"
6448   [(set (match_operand:PTR 0 "memory_operand" "=m")
6449         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6450          UNSPEC_SP_SET))
6451    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6452   ""
6453   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
6454   [(set_attr "length" "12")
6455    (set_attr "type" "multiple")])
6456
6457 (define_expand "stack_protect_test"
6458   [(match_operand 0 "memory_operand")
6459    (match_operand 1 "memory_operand")
6460    (match_operand 2)]
6461   ""
6462 {
6463   rtx result;
6464   machine_mode mode = GET_MODE (operands[0]);
6465
6466   result = gen_reg_rtx(mode);
6467
6468   emit_insn ((mode == DImode
6469               ? gen_stack_protect_test_di
6470               : gen_stack_protect_test_si) (result,
6471                                             operands[0],
6472                                             operands[1]));
6473
6474   if (mode == DImode)
6475     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6476                                     result, const0_rtx, operands[2]));
6477   else
6478     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6479                                     result, const0_rtx, operands[2]));
6480   DONE;
6481 })
6482
6483 (define_insn "stack_protect_test_<mode>"
6484   [(set (match_operand:PTR 0 "register_operand" "=r")
6485         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
6486                      (match_operand:PTR 2 "memory_operand" "m")]
6487          UNSPEC_SP_TEST))
6488    (clobber (match_scratch:PTR 3 "=&r"))]
6489   ""
6490   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
6491   [(set_attr "length" "12")
6492    (set_attr "type" "multiple")])
6493
6494 ;; Write Floating-point Control Register.
6495 (define_insn "set_fpcr"
6496   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
6497   ""
6498   "msr\\tfpcr, %0"
6499   [(set_attr "type" "mrs")])
6500
6501 ;; Read Floating-point Control Register.
6502 (define_insn "get_fpcr"
6503   [(set (match_operand:SI 0 "register_operand" "=r")
6504         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
6505   ""
6506   "mrs\\t%0, fpcr"
6507   [(set_attr "type" "mrs")])
6508
6509 ;; Write Floating-point Status Register.
6510 (define_insn "set_fpsr"
6511   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
6512   ""
6513   "msr\\tfpsr, %0"
6514   [(set_attr "type" "mrs")])
6515
6516 ;; Read Floating-point Status Register.
6517 (define_insn "get_fpsr"
6518   [(set (match_operand:SI 0 "register_operand" "=r")
6519         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
6520   ""
6521   "mrs\\t%0, fpsr"
6522   [(set_attr "type" "mrs")])
6523
6524
6525 ;; Define the subtract-one-and-jump insns so loop.c
6526 ;; knows what to generate.
6527 (define_expand "doloop_end"
6528   [(use (match_operand 0 "" ""))      ; loop pseudo
6529    (use (match_operand 1 "" ""))]     ; label
6530   "optimize > 0 && flag_modulo_sched"
6531 {
6532   rtx s0;
6533   rtx bcomp;
6534   rtx loc_ref;
6535   rtx cc_reg;
6536   rtx insn;
6537   rtx cmp;
6538
6539   /* Currently SMS relies on the do-loop pattern to recognize loops
6540      where (1) the control part consists of all insns defining and/or
6541      using a certain 'count' register and (2) the loop count can be
6542      adjusted by modifying this register prior to the loop.
6543      ??? The possible introduction of a new block to initialize the
6544      new IV can potentially affect branch optimizations.  */
6545
6546   if (GET_MODE (operands[0]) != DImode)
6547     FAIL;
6548
6549   s0 = operands [0];
6550   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
6551
6552   cmp = XVECEXP (PATTERN (insn), 0, 0);
6553   cc_reg = SET_DEST (cmp);
6554   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6555   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6556   emit_jump_insn (gen_rtx_SET (pc_rtx,
6557                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6558                                                      loc_ref, pc_rtx)));
6559   DONE;
6560 })
6561
6562 ;; Track speculation through conditional branches.  We assume that
6563 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
6564 (define_insn "speculation_tracker"
6565   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
6566         (unspec [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
6567          UNSPEC_SPECULATION_TRACKER))]
6568   ""
6569   {
6570     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6571     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
6572     return "";
6573   }
6574   [(set_attr "type" "csel")]
6575 )
6576
6577 ;; Helper for aarch64.c code.
6578 (define_expand "set_clobber_cc"
6579   [(parallel [(set (match_operand 0)
6580                    (match_operand 1))
6581               (clobber (reg:CC CC_REGNUM))])])
6582
6583 ;; Hard speculation barrier.
6584 (define_insn "speculation_barrier"
6585   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
6586   ""
6587   "isb\;dsb\\tsy"
6588   [(set_attr "length" "8")
6589    (set_attr "type" "block")
6590    (set_attr "speculation_barrier" "true")]
6591 )
6592
6593 ;; AdvSIMD Stuff
6594 (include "aarch64-simd.md")
6595
6596 ;; Atomic Operations
6597 (include "atomics.md")
6598
6599 ;; ldp/stp peephole patterns
6600 (include "aarch64-ldpstp.md")
6601
6602 ;; SVE.
6603 (include "aarch64-sve.md")