gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (R17_REGNUM         17)
  42     (R18_REGNUM         18)
  43     (R19_REGNUM         19)
  44     (R20_REGNUM         20)
  45     (R21_REGNUM         21)
  46     (R22_REGNUM         22)
  47     (R23_REGNUM         23)
  48     (R24_REGNUM         24)
  49     (R25_REGNUM         25)
  50     (R26_REGNUM         26)
  51     (R27_REGNUM         27)
  52     (R28_REGNUM         28)
  53     (R29_REGNUM         29)
  54     (R30_REGNUM         30)
  55     (SP_REGNUM          31)
  56     (V0_REGNUM          32)
  57     (V1_REGNUM          33)
  58     (V2_REGNUM          34)
  59     (V3_REGNUM          35)
  60     (V4_REGNUM          36)
  61     (V5_REGNUM          37)
  62     (V6_REGNUM          38)
  63     (V7_REGNUM          39)
  64     (V8_REGNUM          40)
  65     (V9_REGNUM          41)
  66     (V10_REGNUM         42)
  67     (V11_REGNUM         43)
  68     (V12_REGNUM         44)
  69     (V13_REGNUM         45)
  70     (V14_REGNUM         46)
  71     (V15_REGNUM         47)
  72     (V16_REGNUM         48)
  73     (V17_REGNUM         49)
  74     (V18_REGNUM         50)
  75     (V19_REGNUM         51)
  76     (V20_REGNUM         52)
  77     (V21_REGNUM         53)
  78     (V22_REGNUM         54)
  79     (V23_REGNUM         55)
  80     (V24_REGNUM         56)
  81     (V25_REGNUM         57)
  82     (V26_REGNUM         58)
  83     (V27_REGNUM         59)
  84     (V28_REGNUM         60)
  85     (V29_REGNUM         61)
  86     (V30_REGNUM         62)
  87     (V31_REGNUM         63)
  88     (LAST_SAVED_REGNUM  63)
  89     (SFP_REGNUM         64)
  90     (AP_REGNUM          65)
  91     (CC_REGNUM          66)
  92     ;; Defined only to make the DWARF description simpler.
  93     (VG_REGNUM          67)
  94     (P0_REGNUM          68)
  95     (P1_REGNUM          69)
  96     (P2_REGNUM          70)
  97     (P3_REGNUM          71)
  98     (P4_REGNUM          72)
  99     (P5_REGNUM          73)
 100     (P6_REGNUM          74)
 101     (P7_REGNUM          75)
 102     (P8_REGNUM          76)
 103     (P9_REGNUM          77)
 104     (P10_REGNUM         78)
 105     (P11_REGNUM         79)
 106     (P12_REGNUM         80)
 107     (P13_REGNUM         81)
 108     (P14_REGNUM         82)
 109     (P15_REGNUM         83)
 110     ;; Scratch register used by stack clash protection to calculate
 111     ;; SVE CFA offsets during probing.
 112     (STACK_CLASH_SVE_CFA_REGNUM 11)
 113     ;; Scratch registers for prologue/epilogue use.
 114     (EP0_REGNUM         12)
 115     (EP1_REGNUM         13)
 116     ;; A couple of call-clobbered registers that we need to reserve when
 117     ;; tracking speculation this is not ABI, so is subject to change.
 118     (SPECULATION_SCRATCH_REGNUM 14)
 119     (SPECULATION_TRACKER_REGNUM 15)
 120     ;; Scratch registers used in frame layout.
 121     (IP0_REGNUM         16)
 122     (IP1_REGNUM         17)
 123     (FP_REGNUM          29)
 124     (LR_REGNUM          30)
 125   ]
 126 )
 127
 128 (define_c_enum "unspec" [
 129     UNSPEC_AUTIA1716
 130     UNSPEC_AUTIB1716
 131     UNSPEC_AUTIASP
 132     UNSPEC_AUTIBSP
 133     UNSPEC_CASESI
 134     UNSPEC_CRC32B
 135     UNSPEC_CRC32CB
 136     UNSPEC_CRC32CH
 137     UNSPEC_CRC32CW
 138     UNSPEC_CRC32CX
 139     UNSPEC_CRC32H
 140     UNSPEC_CRC32W
 141     UNSPEC_CRC32X
 142     UNSPEC_FCVTZS
 143     UNSPEC_FCVTZU
 144     UNSPEC_FRINT32Z
 145     UNSPEC_FRINT32X
 146     UNSPEC_FRINT64Z
 147     UNSPEC_FRINT64X
 148     UNSPEC_URECPE
 149     UNSPEC_FRECPE
 150     UNSPEC_FRECPS
 151     UNSPEC_FRECPX
 152     UNSPEC_FRINTA
 153     UNSPEC_FRINTI
 154     UNSPEC_FRINTM
 155     UNSPEC_FRINTN
 156     UNSPEC_FRINTP
 157     UNSPEC_FRINTX
 158     UNSPEC_FRINTZ
 159     UNSPEC_GOTSMALLPIC
 160     UNSPEC_GOTSMALLPIC28K
 161     UNSPEC_GOTSMALLTLS
 162     UNSPEC_GOTTINYPIC
 163     UNSPEC_GOTTINYTLS
 164     UNSPEC_LD1
 165     UNSPEC_LD2
 166     UNSPEC_LD2_DREG
 167     UNSPEC_LD2_DUP
 168     UNSPEC_LD3
 169     UNSPEC_LD3_DREG
 170     UNSPEC_LD3_DUP
 171     UNSPEC_LD4
 172     UNSPEC_LD4_DREG
 173     UNSPEC_LD4_DUP
 174     UNSPEC_LD2_LANE
 175     UNSPEC_LD3_LANE
 176     UNSPEC_LD4_LANE
 177     UNSPEC_MB
 178     UNSPEC_NOP
 179     UNSPEC_PACIA1716
 180     UNSPEC_PACIB1716
 181     UNSPEC_PACIASP
 182     UNSPEC_PACIBSP
 183     UNSPEC_PRLG_STK
 184     UNSPEC_REV
 185     UNSPEC_RBIT
 186     UNSPEC_SABAL
 187     UNSPEC_SABDL2
 188     UNSPEC_SADALP
 189     UNSPEC_SCVTF
 190     UNSPEC_SISD_NEG
 191     UNSPEC_SISD_SSHL
 192     UNSPEC_SISD_USHL
 193     UNSPEC_SSHL_2S
 194     UNSPEC_ST1
 195     UNSPEC_ST2
 196     UNSPEC_ST3
 197     UNSPEC_ST4
 198     UNSPEC_ST2_LANE
 199     UNSPEC_ST3_LANE
 200     UNSPEC_ST4_LANE
 201     UNSPEC_TLS
 202     UNSPEC_TLSDESC
 203     UNSPEC_TLSLE12
 204     UNSPEC_TLSLE24
 205     UNSPEC_TLSLE32
 206     UNSPEC_TLSLE48
 207     UNSPEC_UABAL
 208     UNSPEC_UABDL2
 209     UNSPEC_UADALP
 210     UNSPEC_UCVTF
 211     UNSPEC_USHL_2S
 212     UNSPEC_VSTRUCTDUMMY
 213     UNSPEC_SSP_SYSREG
 214     UNSPEC_SP_SET
 215     UNSPEC_SP_TEST
 216     UNSPEC_RSQRT
 217     UNSPEC_RSQRTE
 218     UNSPEC_RSQRTS
 219     UNSPEC_NZCV
 220     UNSPEC_XPACLRI
 221     UNSPEC_LD1_SVE
 222     UNSPEC_ST1_SVE
 223     UNSPEC_LD1RQ
 224     UNSPEC_LD1_GATHER
 225     UNSPEC_ST1_SCATTER
 226     UNSPEC_PRED_X
 227     UNSPEC_PRED_Z
 228     UNSPEC_PTEST
 229     UNSPEC_UNPACKSHI
 230     UNSPEC_UNPACKUHI
 231     UNSPEC_UNPACKSLO
 232     UNSPEC_UNPACKULO
 233     UNSPEC_PACK
 234     UNSPEC_WHILE_LO
 235     UNSPEC_LDN
 236     UNSPEC_STN
 237     UNSPEC_INSR
 238     UNSPEC_CLASTB
 239     UNSPEC_FADDA
 240     UNSPEC_REV_SUBREG
 241     UNSPEC_REINTERPRET
 242     UNSPEC_SPECULATION_TRACKER
 243     UNSPEC_COPYSIGN
 244     UNSPEC_TTEST                ; Represent transaction test.
 245 ])
 246
 247 (define_c_enum "unspecv" [
 248     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 249     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 250     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 251     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 252     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 253     UNSPECV_BLOCKAGE            ; Represent a blockage
 254     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 255     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 256     UNSPECV_BTI_NOARG           ; Represent BTI.
 257     UNSPECV_BTI_C               ; Represent BTI c.
 258     UNSPECV_BTI_J               ; Represent BTI j.
 259     UNSPECV_BTI_JC              ; Represent BTI jc.
 260     UNSPECV_TSTART              ; Represent transaction start.
 261     UNSPECV_TCOMMIT             ; Represent transaction commit.
 262     UNSPECV_TCANCEL             ; Represent transaction cancel.
 263   ]
 264 )
 265
 266 ;; These constants are used as a const_int in various SVE unspecs
 267 ;; to indicate whether the governing predicate is known to be a PTRUE.
 268 (define_constants
 269   [; Indicates that the predicate might not be a PTRUE.
 270    (SVE_MAYBE_NOT_PTRUE 0)
 271
 272    ; Indicates that the predicate is known to be a PTRUE.
 273    (SVE_KNOWN_PTRUE 1)])
 274
 275 ;; These constants are used as a const_int in predicated SVE FP arithmetic
 276 ;; to indicate whether the operation is allowed to make additional lanes
 277 ;; active without worrying about the effect on faulting behavior.
 278 (define_constants
 279   [; Indicates either that all lanes are active or that the instruction may
 280    ; operate on inactive inputs even if doing so could induce a fault.
 281    (SVE_RELAXED_GP 0)
 282
 283    ; Indicates that some lanes might be inactive and that the instruction
 284    ; must not operate on inactive inputs if doing so could induce a fault.
 285    (SVE_STRICT_GP 1)])
 286
 287 ;; If further include files are added the defintion of MD_INCLUDES
 288 ;; must be updated.
 289
 290 (include "constraints.md")
 291 (include "predicates.md")
 292 (include "iterators.md")
 293
 294 ;; -------------------------------------------------------------------
 295 ;; Instruction types and attributes
 296 ;; -------------------------------------------------------------------
 297
 298 ; The "type" attribute is included here from AArch32 backend to be able
 299 ; to share pipeline descriptions.
 300 (include "../arm/types.md")
 301
 302 ;; It is important to set the fp or simd attributes to yes when a pattern
 303 ;; alternative uses the FP or SIMD register files, usually signified by use of
 304 ;; the 'w' constraint.  This will ensure that the alternative will be
 305 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 306 ;; architecture extensions.  If all the alternatives in a pattern use the
 307 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 308 ;; or TARGET_SIMD.
 309
 310 ;; Attributes of the architecture required to support the instruction (or
 311 ;; alternative). This attribute is used to compute attribute "enabled", use type
 312 ;; "any" to enable an alternative in all cases.
 313
 314 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
 315
 316 (define_enum_attr "arch" "arches" (const_string "any"))
 317
 318 ;; [For compatibility with Arm in pipeline models]
 319 ;; Attribute that specifies whether or not the instruction touches fp
 320 ;; registers.
 321 ;; Note that this attribute is not used anywhere in either the arm or aarch64
 322 ;; backends except in the scheduling description for xgene1.  In that
 323 ;; scheduling description this attribute is used to subclass the load_4 and
 324 ;; load_8 types.
 325 (define_attr "fp" "no,yes"
 326   (if_then_else
 327     (eq_attr "arch" "fp")
 328     (const_string "yes")
 329     (const_string "no")))
 330
 331 (define_attr "arch_enabled" "no,yes"
 332   (if_then_else
 333     (ior
 334         (eq_attr "arch" "any")
 335
 336         (and (eq_attr "arch" "rcpc8_4")
 337              (match_test "AARCH64_ISA_RCPC8_4"))
 338
 339         (and (eq_attr "arch" "fp")
 340              (match_test "TARGET_FLOAT"))
 341
 342         (and (eq_attr "arch" "simd")
 343              (match_test "TARGET_SIMD"))
 344
 345         (and (eq_attr "arch" "fp16")
 346              (match_test "TARGET_FP_F16INST"))
 347
 348         (and (eq_attr "arch" "sve")
 349              (match_test "TARGET_SVE")))
 350     (const_string "yes")
 351     (const_string "no")))
 352
 353 ;; Attribute that controls whether an alternative is enabled or not.
 354 ;; Currently it is only used to disable alternatives which touch fp or simd
 355 ;; registers when -mgeneral-regs-only is specified or to require a special
 356 ;; architecture support.
 357 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
 358
 359 ;; Attribute that specifies whether we are dealing with a branch to a
 360 ;; label that is far away, i.e. further away than the maximum/minimum
 361 ;; representable in a signed 21-bits number.
 362 ;; 0 :=: no
 363 ;; 1 :=: yes
 364 (define_attr "far_branch" "" (const_int 0))
 365
 366 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 367 (define_attr "movprfx" "no,yes" (const_string "no"))
 368
 369 (define_attr "length" ""
 370   (cond [(eq_attr "movprfx" "yes")
 371            (const_int 8)
 372         ] (const_int 4)))
 373
 374 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 375 ;; no predicated insns.
 376 (define_attr "predicated" "yes,no" (const_string "no"))
 377
 378 ;; Set to true on an insn that requires the speculation tracking state to be
 379 ;; in the tracking register before the insn issues.  Otherwise the compiler
 380 ;; may chose to hold the tracking state encoded in SP.
 381 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 382
 383 ;; -------------------------------------------------------------------
 384 ;; Pipeline descriptions and scheduling
 385 ;; -------------------------------------------------------------------
 386
 387 ;; Processor types.
 388 (include "aarch64-tune.md")
 389
 390 ;; Scheduling
 391 (include "../arm/cortex-a53.md")
 392 (include "../arm/cortex-a57.md")
 393 (include "../arm/exynos-m1.md")
 394 (include "falkor.md")
 395 (include "saphira.md")
 396 (include "thunderx.md")
 397 (include "../arm/xgene1.md")
 398 (include "thunderx2t99.md")
 399 (include "tsv110.md")
 400
 401 ;; -------------------------------------------------------------------
 402 ;; Jumps and other miscellaneous insns
 403 ;; -------------------------------------------------------------------
 404
 405 (define_insn "indirect_jump"
 406   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 407   ""
 408   "br\\t%0"
 409   [(set_attr "type" "branch")]
 410 )
 411
 412 (define_insn "jump"
 413   [(set (pc) (label_ref (match_operand 0 "" "")))]
 414   ""
 415   "b\\t%l0"
 416   [(set_attr "type" "branch")]
 417 )
 418
 419 (define_expand "cbranch<mode>4"
 420   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 421                             [(match_operand:GPI 1 "register_operand")
 422                              (match_operand:GPI 2 "aarch64_plus_operand")])
 423                            (label_ref (match_operand 3 "" ""))
 424                            (pc)))]
 425   ""
 426   "
 427   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 428                                          operands[2]);
 429   operands[2] = const0_rtx;
 430   "
 431 )
 432
 433 (define_expand "cbranch<mode>4"
 434   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 435                             [(match_operand:GPF 1 "register_operand")
 436                              (match_operand:GPF 2 "aarch64_fp_compare_operand")])
 437                            (label_ref (match_operand 3 "" ""))
 438                            (pc)))]
 439   ""
 440   "
 441   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 442                                          operands[2]);
 443   operands[2] = const0_rtx;
 444   "
 445 )
 446
 447 (define_expand "cbranchcc4"
 448   [(set (pc) (if_then_else
 449               (match_operator 0 "aarch64_comparison_operator"
 450                [(match_operand 1 "cc_register")
 451                 (match_operand 2 "const0_operand")])
 452               (label_ref (match_operand 3 "" ""))
 453               (pc)))]
 454   ""
 455   "")
 456
 457 (define_insn "ccmp<mode>"
 458   [(set (match_operand:CC 1 "cc_register" "")
 459         (if_then_else:CC
 460           (match_operator 4 "aarch64_comparison_operator"
 461            [(match_operand 0 "cc_register" "")
 462             (const_int 0)])
 463           (compare:CC
 464             (match_operand:GPI 2 "register_operand" "r,r,r")
 465             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 466           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 467   ""
 468   "@
 469    ccmp\\t%<w>2, %<w>3, %k5, %m4
 470    ccmp\\t%<w>2, %3, %k5, %m4
 471    ccmn\\t%<w>2, #%n3, %k5, %m4"
 472   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 473 )
 474
 475 (define_insn "fccmp<mode>"
 476   [(set (match_operand:CCFP 1 "cc_register" "")
 477         (if_then_else:CCFP
 478           (match_operator 4 "aarch64_comparison_operator"
 479            [(match_operand 0 "cc_register" "")
 480             (const_int 0)])
 481           (compare:CCFP
 482             (match_operand:GPF 2 "register_operand" "w")
 483             (match_operand:GPF 3 "register_operand" "w"))
 484           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 485   "TARGET_FLOAT"
 486   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 487   [(set_attr "type" "fccmp<s>")]
 488 )
 489
 490 (define_insn "fccmpe<mode>"
 491   [(set (match_operand:CCFPE 1 "cc_register" "")
 492          (if_then_else:CCFPE
 493           (match_operator 4 "aarch64_comparison_operator"
 494            [(match_operand 0 "cc_register" "")
 495           (const_int 0)])
 496            (compare:CCFPE
 497             (match_operand:GPF 2 "register_operand" "w")
 498             (match_operand:GPF 3 "register_operand" "w"))
 499           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 500   "TARGET_FLOAT"
 501   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 502   [(set_attr "type" "fccmp<s>")]
 503 )
 504
 505 ;; Expansion of signed mod by a power of 2 using CSNEG.
 506 ;; For x0 % n where n is a power of 2 produce:
 507 ;; negs   x1, x0
 508 ;; and    x0, x0, #(n - 1)
 509 ;; and    x1, x1, #(n - 1)
 510 ;; csneg  x0, x0, x1, mi
 511
 512 (define_expand "mod<mode>3"
 513   [(match_operand:GPI 0 "register_operand")
 514    (match_operand:GPI 1 "register_operand")
 515    (match_operand:GPI 2 "const_int_operand")]
 516   ""
 517   {
 518     HOST_WIDE_INT val = INTVAL (operands[2]);
 519
 520     if (val <= 0
 521        || exact_log2 (val) <= 0
 522        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 523       FAIL;
 524
 525     rtx mask = GEN_INT (val - 1);
 526
 527     /* In the special case of x0 % 2 we can do the even shorter:
 528         cmp    x0, xzr
 529         and    x0, x0, 1
 530         cneg   x0, x0, lt.  */
 531     if (val == 2)
 532       {
 533         rtx masked = gen_reg_rtx (<MODE>mode);
 534         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 535         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 536         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 537         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 538         DONE;
 539       }
 540
 541     rtx neg_op = gen_reg_rtx (<MODE>mode);
 542     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 543
 544     /* Extract the condition register and mode.  */
 545     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 546     rtx cc_reg = SET_DEST (cmp);
 547     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 548
 549     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 550     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 551
 552     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 553     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 554
 555     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 556                                        masked_neg, masked_pos));
 557     DONE;
 558   }
 559 )
 560
 561 (define_insn "condjump"
 562   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 563                             [(match_operand 1 "cc_register" "") (const_int 0)])
 564                            (label_ref (match_operand 2 "" ""))
 565                            (pc)))]
 566   ""
 567   {
 568     /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
 569        but the "." is required for SVE conditions.  */
 570     bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
 571     if (get_attr_length (insn) == 8)
 572       return aarch64_gen_far_branch (operands, 2, "Lbcond",
 573                                      use_dot_p ? "b.%M0\\t" : "b%M0\\t");
 574     else
 575       return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
 576   }
 577   [(set_attr "type" "branch")
 578    (set (attr "length")
 579         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 580                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 581                       (const_int 4)
 582                       (const_int 8)))
 583    (set (attr "far_branch")
 584         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 585                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 586                       (const_int 0)
 587                       (const_int 1)))]
 588 )
 589
 590 ;; For a 24-bit immediate CST we can optimize the compare for equality
 591 ;; and branch sequence from:
 592 ;;      mov     x0, #imm1
 593 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 594 ;;      cmp     x1, x0
 595 ;;      b<ne,eq> .Label
 596 ;; into the shorter:
 597 ;;      sub     x0, x1, #(CST & 0xfff000)
 598 ;;      subs    x0, x0, #(CST & 0x000fff)
 599 ;;      b<ne,eq> .Label
 600 (define_insn_and_split "*compare_condjump<GPI:mode>"
 601   [(set (pc) (if_then_else (EQL
 602                               (match_operand:GPI 0 "register_operand" "r")
 603                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 604                            (label_ref:P (match_operand 2 "" ""))
 605                            (pc)))]
 606   "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
 607    && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
 608    && !reload_completed"
 609   "#"
 610   "&& true"
 611   [(const_int 0)]
 612   {
 613     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 614     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 615     rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
 616     emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 617     emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 618     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 619     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
 620                                   cc_reg, const0_rtx);
 621     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 622     DONE;
 623   }
 624 )
 625
 626 (define_expand "casesi"
 627   [(match_operand:SI 0 "register_operand")      ; Index
 628    (match_operand:SI 1 "const_int_operand")     ; Lower bound
 629    (match_operand:SI 2 "const_int_operand")     ; Total range
 630    (match_operand:DI 3 "" "")                   ; Table label
 631    (match_operand:DI 4 "" "")]                  ; Out of range label
 632   ""
 633   {
 634     if (operands[1] != const0_rtx)
 635       {
 636         rtx reg = gen_reg_rtx (SImode);
 637
 638         /* Canonical RTL says that if you have:
 639
 640            (minus (X) (CONST))
 641
 642            then this should be emitted as:
 643
 644            (plus (X) (-CONST))
 645
 646            The use of trunc_int_for_mode ensures that the resulting
 647            constant can be represented in SImode, this is important
 648            for the corner case where operand[1] is INT_MIN.  */
 649
 650         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 651
 652         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 653               (operands[1], SImode))
 654           operands[1] = force_reg (SImode, operands[1]);
 655         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 656         operands[0] = reg;
 657       }
 658
 659     if (!aarch64_plus_operand (operands[2], SImode))
 660       operands[2] = force_reg (SImode, operands[2]);
 661     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 662                                                  const0_rtx),
 663                                     operands[0], operands[2], operands[4]));
 664
 665     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 666     operands[2]
 667       = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
 668                         UNSPEC_CASESI);
 669     operands[2] = gen_rtx_MEM (DImode, operands[2]);
 670     MEM_READONLY_P (operands[2]) = 1;
 671     MEM_NOTRAP_P (operands[2]) = 1;
 672     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
 673     DONE;
 674   }
 675 )
 676
 677 (define_expand "casesi_dispatch"
 678   [(parallel
 679     [(set (pc) (match_operand:DI 0 ""))
 680      (clobber (reg:CC CC_REGNUM))
 681      (clobber (match_scratch:DI 2))
 682      (clobber (match_scratch:DI 3))
 683      (use (label_ref:DI (match_operand 1 "")))])]
 684   "")
 685
 686 (define_insn "*casesi_dispatch"
 687   [(parallel
 688     [(set (pc)
 689           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 690                            (match_operand:SI 1 "register_operand" "r")]
 691                         UNSPEC_CASESI)))
 692      (clobber (reg:CC CC_REGNUM))
 693      (clobber (match_scratch:DI 3 "=r"))
 694      (clobber (match_scratch:DI 4 "=r"))
 695      (use (label_ref:DI (match_operand 2 "" "")))])]
 696   ""
 697   "*
 698   return aarch64_output_casesi (operands);
 699   "
 700   [(set_attr "length" "16")
 701    (set_attr "type" "branch")]
 702 )
 703
 704 (define_insn "nop"
 705   [(unspec[(const_int 0)] UNSPEC_NOP)]
 706   ""
 707   "nop"
 708   [(set_attr "type" "no_insn")]
 709 )
 710
 711 (define_insn "prefetch"
 712   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 713             (match_operand:QI 1 "const_int_operand" "")
 714             (match_operand:QI 2 "const_int_operand" ""))]
 715   ""
 716   {
 717     const char * pftype[2][4] =
 718     {
 719       {"prfm\\tPLDL1STRM, %0",
 720        "prfm\\tPLDL3KEEP, %0",
 721        "prfm\\tPLDL2KEEP, %0",
 722        "prfm\\tPLDL1KEEP, %0"},
 723       {"prfm\\tPSTL1STRM, %0",
 724        "prfm\\tPSTL3KEEP, %0",
 725        "prfm\\tPSTL2KEEP, %0",
 726        "prfm\\tPSTL1KEEP, %0"},
 727     };
 728
 729     int locality = INTVAL (operands[2]);
 730
 731     gcc_assert (IN_RANGE (locality, 0, 3));
 732
 733     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 734        the address into a DImode MEM so that aarch64_print_operand knows
 735        how to print it.  */
 736     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 737     return pftype[INTVAL(operands[1])][locality];
 738   }
 739   [(set_attr "type" "load_4")]
 740 )
 741
 742 (define_insn "trap"
 743   [(trap_if (const_int 1) (const_int 8))]
 744   ""
 745   "brk #1000"
 746   [(set_attr "type" "trap")])
 747
 748 (define_expand "prologue"
 749   [(clobber (const_int 0))]
 750   ""
 751   "
 752   aarch64_expand_prologue ();
 753   DONE;
 754   "
 755 )
 756
 757 (define_expand "epilogue"
 758   [(clobber (const_int 0))]
 759   ""
 760   "
 761   aarch64_expand_epilogue (false);
 762   DONE;
 763   "
 764 )
 765
 766 (define_expand "sibcall_epilogue"
 767   [(clobber (const_int 0))]
 768   ""
 769   "
 770   aarch64_expand_epilogue (true);
 771   DONE;
 772   "
 773 )
 774
 775 (define_insn "*do_return"
 776   [(return)]
 777   ""
 778   {
 779     if (aarch64_return_address_signing_enabled ()
 780         && TARGET_ARMV8_3
 781         && !crtl->calls_eh_return)
 782       {
 783         if (aarch64_ra_sign_key == AARCH64_KEY_B)
 784           return "retab";
 785         else
 786           return "retaa";
 787       }
 788     return "ret";
 789   }
 790   [(set_attr "type" "branch")]
 791 )
 792
 793 (define_expand "return"
 794   [(simple_return)]
 795   "aarch64_use_return_insn_p ()"
 796   ""
 797 )
 798
 799 (define_insn "simple_return"
 800   [(simple_return)]
 801   "aarch64_use_simple_return_insn_p ()"
 802   "ret"
 803   [(set_attr "type" "branch")]
 804 )
 805
 806 (define_insn "*cb<optab><mode>1"
 807   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 808                                 (const_int 0))
 809                            (label_ref (match_operand 1 "" ""))
 810                            (pc)))]
 811   "!aarch64_track_speculation"
 812   {
 813     if (get_attr_length (insn) == 8)
 814       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 815     else
 816       return "<cbz>\\t%<w>0, %l1";
 817   }
 818   [(set_attr "type" "branch")
 819    (set (attr "length")
 820         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 821                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 822                       (const_int 4)
 823                       (const_int 8)))
 824    (set (attr "far_branch")
 825         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 826                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 827                       (const_int 0)
 828                       (const_int 1)))]
 829 )
 830
 831 (define_insn "*tb<optab><mode>1"
 832   [(set (pc) (if_then_else
 833               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 834                                     (const_int 1)
 835                                     (match_operand 1
 836                                       "aarch64_simd_shift_imm_<mode>" "n"))
 837                    (const_int 0))
 838              (label_ref (match_operand 2 "" ""))
 839              (pc)))
 840    (clobber (reg:CC CC_REGNUM))]
 841   "!aarch64_track_speculation"
 842   {
 843     if (get_attr_length (insn) == 8)
 844       {
 845         if (get_attr_far_branch (insn) == 1)
 846           return aarch64_gen_far_branch (operands, 2, "Ltb",
 847                                          "<inv_tb>\\t%<w>0, %1, ");
 848         else
 849           {
 850             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 851             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 852           }
 853       }
 854     else
 855       return "<tbz>\t%<w>0, %1, %l2";
 856   }
 857   [(set_attr "type" "branch")
 858    (set (attr "length")
 859         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 860                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 861                       (const_int 4)
 862                       (const_int 8)))
 863    (set (attr "far_branch")
 864         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 865                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 866                       (const_int 0)
 867                       (const_int 1)))]
 868
 869 )
 870
 871 (define_insn "*cb<optab><mode>1"
 872   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 873                                  (const_int 0))
 874                            (label_ref (match_operand 1 "" ""))
 875                            (pc)))
 876    (clobber (reg:CC CC_REGNUM))]
 877   "!aarch64_track_speculation"
 878   {
 879     if (get_attr_length (insn) == 8)
 880       {
 881         if (get_attr_far_branch (insn) == 1)
 882           return aarch64_gen_far_branch (operands, 1, "Ltb",
 883                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 884         else
 885           {
 886             char buf[64];
 887             uint64_t val = ((uint64_t) 1)
 888                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 889             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 890             output_asm_insn (buf, operands);
 891             return "<bcond>\t%l1";
 892           }
 893       }
 894     else
 895       return "<tbz>\t%<w>0, <sizem1>, %l1";
 896   }
 897   [(set_attr "type" "branch")
 898    (set (attr "length")
 899         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 900                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 901                       (const_int 4)
 902                       (const_int 8)))
 903    (set (attr "far_branch")
 904         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 905                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 906                       (const_int 0)
 907                       (const_int 1)))]
 908 )
 909
 910 ;; -------------------------------------------------------------------
 911 ;; Subroutine calls and sibcalls
 912 ;; -------------------------------------------------------------------
 913
 914 (define_expand "call"
 915   [(parallel [(call (match_operand 0 "memory_operand")
 916                     (match_operand 1 "general_operand"))
 917               (use (match_operand 2 "" ""))
 918               (clobber (reg:DI LR_REGNUM))])]
 919   ""
 920   "
 921   {
 922     aarch64_expand_call (NULL_RTX, operands[0], false);
 923     DONE;
 924   }"
 925 )
 926
 927 (define_insn "*call_insn"
 928   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 929          (match_operand 1 "" ""))
 930    (clobber (reg:DI LR_REGNUM))]
 931   ""
 932   "@
 933   blr\\t%0
 934   bl\\t%c0"
 935   [(set_attr "type" "call, call")]
 936 )
 937
 938 (define_expand "call_value"
 939   [(parallel [(set (match_operand 0 "" "")
 940                    (call (match_operand 1 "memory_operand")
 941                          (match_operand 2 "general_operand")))
 942               (use (match_operand 3 "" ""))
 943               (clobber (reg:DI LR_REGNUM))])]
 944   ""
 945   "
 946   {
 947     aarch64_expand_call (operands[0], operands[1], false);
 948     DONE;
 949   }"
 950 )
 951
 952 (define_insn "*call_value_insn"
 953   [(set (match_operand 0 "" "")
 954         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 955                       (match_operand 2 "" "")))
 956    (clobber (reg:DI LR_REGNUM))]
 957   ""
 958   "@
 959   blr\\t%1
 960   bl\\t%c1"
 961   [(set_attr "type" "call, call")]
 962 )
 963
 964 (define_expand "sibcall"
 965   [(parallel [(call (match_operand 0 "memory_operand")
 966                     (match_operand 1 "general_operand"))
 967               (return)
 968               (use (match_operand 2 "" ""))])]
 969   ""
 970   {
 971     aarch64_expand_call (NULL_RTX, operands[0], true);
 972     DONE;
 973   }
 974 )
 975
 976 (define_expand "sibcall_value"
 977   [(parallel [(set (match_operand 0 "" "")
 978                    (call (match_operand 1 "memory_operand")
 979                          (match_operand 2 "general_operand")))
 980               (return)
 981               (use (match_operand 3 "" ""))])]
 982   ""
 983   {
 984     aarch64_expand_call (operands[0], operands[1], true);
 985     DONE;
 986   }
 987 )
 988
 989 (define_insn "*sibcall_insn"
 990   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 991          (match_operand 1 "" ""))
 992    (return)]
 993   "SIBLING_CALL_P (insn)"
 994   "@
 995    br\\t%0
 996    b\\t%c0"
 997   [(set_attr "type" "branch, branch")]
 998 )
 999
1000 (define_insn "*sibcall_value_insn"
1001   [(set (match_operand 0 "" "")
1002         (call (mem:DI
1003                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1004               (match_operand 2 "" "")))
1005    (return)]
1006   "SIBLING_CALL_P (insn)"
1007   "@
1008    br\\t%1
1009    b\\t%c1"
1010   [(set_attr "type" "branch, branch")]
1011 )
1012
1013 ;; Call subroutine returning any type.
1014
1015 (define_expand "untyped_call"
1016   [(parallel [(call (match_operand 0 "")
1017                     (const_int 0))
1018               (match_operand 1 "")
1019               (match_operand 2 "")])]
1020   ""
1021 {
1022   int i;
1023
1024   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
1025
1026   for (i = 0; i < XVECLEN (operands[2], 0); i++)
1027     {
1028       rtx set = XVECEXP (operands[2], 0, i);
1029       emit_move_insn (SET_DEST (set), SET_SRC (set));
1030     }
1031
1032   /* The optimizer does not know that the call sets the function value
1033      registers we stored in the result block.  We avoid problems by
1034      claiming that all hard registers are used and clobbered at this
1035      point.  */
1036   emit_insn (gen_blockage ());
1037   DONE;
1038 })
1039
1040 ;; -------------------------------------------------------------------
1041 ;; Moves
1042 ;; -------------------------------------------------------------------
1043
1044 (define_expand "mov<mode>"
1045   [(set (match_operand:SHORT 0 "nonimmediate_operand")
1046         (match_operand:SHORT 1 "general_operand"))]
1047   ""
1048   "
1049     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1050       operands[1] = force_reg (<MODE>mode, operands[1]);
1051
1052     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1053       {
1054         aarch64_expand_mov_immediate (operands[0], operands[1]);
1055         DONE;
1056       }
1057   "
1058 )
1059
1060 (define_insn "*mov<mode>_aarch64"
1061   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
1062         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1063   "(register_operand (operands[0], <MODE>mode)
1064     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1065 {
1066    switch (which_alternative)
1067      {
1068      case 0:
1069        return "mov\t%w0, %w1";
1070      case 1:
1071        return "mov\t%w0, %1";
1072      case 2:
1073        return aarch64_output_scalar_simd_mov_immediate (operands[1],
1074                                                         <MODE>mode);
1075      case 3:
1076        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1077      case 4:
1078        return "ldr<size>\t%w0, %1";
1079      case 5:
1080        return "ldr\t%<size>0, %1";
1081      case 6:
1082        return "str<size>\t%w1, %0";
1083      case 7:
1084        return "str\t%<size>1, %0";
1085      case 8:
1086        return "umov\t%w0, %1.<v>[0]";
1087      case 9:
1088        return "dup\t%0.<Vallxd>, %w1";
1089      case 10:
1090        return "dup\t%<Vetype>0, %1.<v>[0]";
1091      default:
1092        gcc_unreachable ();
1093      }
1094 }
1095   ;; The "mov_imm" type for CNT is just a placeholder.
1096   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1097                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1098    (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1099 )
1100
1101 (define_expand "mov<mode>"
1102   [(set (match_operand:GPI 0 "nonimmediate_operand")
1103         (match_operand:GPI 1 "general_operand"))]
1104   ""
1105   "
1106     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1107         && <MODE>mode == DImode
1108         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1109       DONE;
1110
1111     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1112       operands[1] = force_reg (<MODE>mode, operands[1]);
1113
1114     /* FIXME: RR we still need to fix up what we are doing with
1115        symbol_refs and other types of constants.  */
1116     if (CONSTANT_P (operands[1])
1117         && !CONST_INT_P (operands[1]))
1118      {
1119        aarch64_expand_mov_immediate (operands[0], operands[1]);
1120        DONE;
1121      }
1122   "
1123 )
1124
1125 (define_insn_and_split "*movsi_aarch64"
1126   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1127         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1128   "(register_operand (operands[0], SImode)
1129     || aarch64_reg_or_zero (operands[1], SImode))"
1130   "@
1131    mov\\t%w0, %w1
1132    mov\\t%w0, %w1
1133    mov\\t%w0, %w1
1134    mov\\t%w0, %1
1135    #
1136    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1137    ldr\\t%w0, %1
1138    ldr\\t%s0, %1
1139    str\\t%w1, %0
1140    str\\t%s1, %0
1141    adr\\t%x0, %c1
1142    adrp\\t%x0, %A1
1143    fmov\\t%s0, %w1
1144    fmov\\t%w0, %s1
1145    fmov\\t%s0, %s1
1146    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1147   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1148     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1149    [(const_int 0)]
1150    "{
1151        aarch64_expand_mov_immediate (operands[0], operands[1]);
1152        DONE;
1153     }"
1154   ;; The "mov_imm" type for CNT is just a placeholder.
1155   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1156                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1157    (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1158 )
1159
1160 (define_insn_and_split "*movdi_aarch64"
1161   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1162         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1163   "(register_operand (operands[0], DImode)
1164     || aarch64_reg_or_zero (operands[1], DImode))"
1165   "@
1166    mov\\t%x0, %x1
1167    mov\\t%0, %x1
1168    mov\\t%x0, %1
1169    mov\\t%x0, %1
1170    mov\\t%w0, %1
1171    #
1172    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1173    ldr\\t%x0, %1
1174    ldr\\t%d0, %1
1175    str\\t%x1, %0
1176    str\\t%d1, %0
1177    adr\\t%x0, %c1
1178    adrp\\t%x0, %A1
1179    fmov\\t%d0, %x1
1180    fmov\\t%x0, %d1
1181    fmov\\t%d0, %d1
1182    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1183    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1184     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1185    [(const_int 0)]
1186    "{
1187        aarch64_expand_mov_immediate (operands[0], operands[1]);
1188        DONE;
1189     }"
1190   ;; The "mov_imm" type for CNTD is just a placeholder.
1191   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1192                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1193                      neon_move")
1194    (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1195 )
1196
1197 (define_insn "insv_imm<mode>"
1198   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1199                           (const_int 16)
1200                           (match_operand:GPI 1 "const_int_operand" "n"))
1201         (match_operand:GPI 2 "const_int_operand" "n"))]
1202   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1203    && UINTVAL (operands[1]) % 16 == 0"
1204   "movk\\t%<w>0, %X2, lsl %1"
1205   [(set_attr "type" "mov_imm")]
1206 )
1207
1208 (define_expand "movti"
1209   [(set (match_operand:TI 0 "nonimmediate_operand")
1210         (match_operand:TI 1 "general_operand"))]
1211   ""
1212   "
1213     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1214       operands[1] = force_reg (TImode, operands[1]);
1215
1216     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1217       {
1218         emit_move_insn (gen_lowpart (DImode, operands[0]),
1219                         gen_lowpart (DImode, operands[1]));
1220         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1221         DONE;
1222       }
1223   "
1224 )
1225
1226 (define_insn "*movti_aarch64"
1227   [(set (match_operand:TI 0
1228          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1229         (match_operand:TI 1
1230          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1231   "(register_operand (operands[0], TImode)
1232     || aarch64_reg_or_zero (operands[1], TImode))"
1233   "@
1234    #
1235    #
1236    #
1237    mov\\t%0.16b, %1.16b
1238    ldp\\t%0, %H0, %1
1239    stp\\t%1, %H1, %0
1240    stp\\txzr, xzr, %0
1241    ldr\\t%q0, %1
1242    str\\t%q1, %0"
1243   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1244                              load_16,store_16,store_16,\
1245                              load_16,store_16")
1246    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1247    (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1248 )
1249
1250 ;; Split a TImode register-register or register-immediate move into
1251 ;; its component DImode pieces, taking care to handle overlapping
1252 ;; source and dest registers.
1253 (define_split
1254    [(set (match_operand:TI 0 "register_operand" "")
1255          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1256   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1257   [(const_int 0)]
1258 {
1259   aarch64_split_128bit_move (operands[0], operands[1]);
1260   DONE;
1261 })
1262
1263 (define_expand "mov<mode>"
1264   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1265         (match_operand:GPF_TF_F16 1 "general_operand"))]
1266   ""
1267   {
1268     if (!TARGET_FLOAT)
1269       {
1270         aarch64_err_no_fpadvsimd (<MODE>mode);
1271         FAIL;
1272       }
1273
1274     if (GET_CODE (operands[0]) == MEM
1275         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1276               && aarch64_float_const_zero_rtx_p (operands[1])))
1277       operands[1] = force_reg (<MODE>mode, operands[1]);
1278   }
1279 )
1280
1281 (define_insn "*movhf_aarch64"
1282   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1283         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1284   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1285     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1286   "@
1287    movi\\t%0.4h, #0
1288    fmov\\t%h0, %w1
1289    dup\\t%w0.4h, %w1
1290    umov\\t%w0, %1.h[0]
1291    mov\\t%0.h[0], %1.h[0]
1292    fmov\\t%h0, %1
1293    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1294    ldr\\t%h0, %1
1295    str\\t%h1, %0
1296    ldrh\\t%w0, %1
1297    strh\\t%w1, %0
1298    mov\\t%w0, %w1"
1299   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1300                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1301    (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1302 )
1303
1304 (define_insn "*movsf_aarch64"
1305   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1306         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1307   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1308     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1309   "@
1310    movi\\t%0.2s, #0
1311    fmov\\t%s0, %w1
1312    fmov\\t%w0, %s1
1313    fmov\\t%s0, %s1
1314    fmov\\t%s0, %1
1315    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1316    ldr\\t%s0, %1
1317    str\\t%s1, %0
1318    ldr\\t%w0, %1
1319    str\\t%w1, %0
1320    mov\\t%w0, %w1
1321    mov\\t%w0, %1"
1322   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1323                      f_loads,f_stores,load_4,store_4,mov_reg,\
1324                      fconsts")
1325    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1326 )
1327
1328 (define_insn "*movdf_aarch64"
1329   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1330         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1331   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1332     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1333   "@
1334    movi\\t%d0, #0
1335    fmov\\t%d0, %x1
1336    fmov\\t%x0, %d1
1337    fmov\\t%d0, %d1
1338    fmov\\t%d0, %1
1339    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1340    ldr\\t%d0, %1
1341    str\\t%d1, %0
1342    ldr\\t%x0, %1
1343    str\\t%x1, %0
1344    mov\\t%x0, %x1
1345    mov\\t%x0, %1"
1346   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1347                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1348                      fconstd")
1349    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1350 )
1351
1352 (define_split
1353   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1354         (match_operand:GPF_HF 1 "general_operand"))]
1355   "can_create_pseudo_p ()
1356    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1357    && !aarch64_float_const_representable_p (operands[1])
1358    &&  aarch64_float_const_rtx_p (operands[1])"
1359   [(const_int 0)]
1360   {
1361     unsigned HOST_WIDE_INT ival;
1362     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1363       FAIL;
1364
1365     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1366     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1367     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1368     DONE;
1369   }
1370 )
1371
1372 (define_insn "*movtf_aarch64"
1373   [(set (match_operand:TF 0
1374          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1375         (match_operand:TF 1
1376          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1377   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1378     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1379   "@
1380    mov\\t%0.16b, %1.16b
1381    #
1382    #
1383    #
1384    movi\\t%0.2d, #0
1385    fmov\\t%s0, wzr
1386    ldr\\t%q0, %1
1387    str\\t%q1, %0
1388    ldp\\t%0, %H0, %1
1389    stp\\t%1, %H1, %0
1390    stp\\txzr, xzr, %0"
1391   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1392                      f_loadd,f_stored,load_16,store_16,store_16")
1393    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1394    (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1395 )
1396
1397 (define_split
1398    [(set (match_operand:TF 0 "register_operand" "")
1399          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1400   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1401   [(const_int 0)]
1402   {
1403     aarch64_split_128bit_move (operands[0], operands[1]);
1404     DONE;
1405   }
1406 )
1407
1408 ;; 0 is dst
1409 ;; 1 is src
1410 ;; 2 is size of copy in bytes
1411 ;; 3 is alignment
1412
1413 (define_expand "cpymemdi"
1414   [(match_operand:BLK 0 "memory_operand")
1415    (match_operand:BLK 1 "memory_operand")
1416    (match_operand:DI 2 "immediate_operand")
1417    (match_operand:DI 3 "immediate_operand")]
1418    "!STRICT_ALIGNMENT"
1419 {
1420   if (aarch64_expand_cpymem (operands))
1421     DONE;
1422   FAIL;
1423 }
1424 )
1425
1426 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1427 ;; fairly lax checking on the second memory operation.
1428 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1429   [(set (match_operand:SX 0 "register_operand" "=r,w")
1430         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1431    (set (match_operand:SX2 2 "register_operand" "=r,w")
1432         (match_operand:SX2 3 "memory_operand" "m,m"))]
1433    "rtx_equal_p (XEXP (operands[3], 0),
1434                  plus_constant (Pmode,
1435                                 XEXP (operands[1], 0),
1436                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1437   "@
1438    ldp\\t%w0, %w2, %1
1439    ldp\\t%s0, %s2, %1"
1440   [(set_attr "type" "load_8,neon_load1_2reg")
1441    (set_attr "arch" "*,fp")]
1442 )
1443
1444 ;; Storing different modes that can still be merged
1445 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1446   [(set (match_operand:DX 0 "register_operand" "=r,w")
1447         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1448    (set (match_operand:DX2 2 "register_operand" "=r,w")
1449         (match_operand:DX2 3 "memory_operand" "m,m"))]
1450    "rtx_equal_p (XEXP (operands[3], 0),
1451                  plus_constant (Pmode,
1452                                 XEXP (operands[1], 0),
1453                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1454   "@
1455    ldp\\t%x0, %x2, %1
1456    ldp\\t%d0, %d2, %1"
1457   [(set_attr "type" "load_16,neon_load1_2reg")
1458    (set_attr "arch" "*,fp")]
1459 )
1460
1461 (define_insn "load_pair_dw_tftf"
1462   [(set (match_operand:TF 0 "register_operand" "=w")
1463         (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1464    (set (match_operand:TF 2 "register_operand" "=w")
1465         (match_operand:TF 3 "memory_operand" "m"))]
1466    "TARGET_SIMD
1467     && rtx_equal_p (XEXP (operands[3], 0),
1468                     plus_constant (Pmode,
1469                                    XEXP (operands[1], 0),
1470                                    GET_MODE_SIZE (TFmode)))"
1471   "ldp\\t%q0, %q2, %1"
1472   [(set_attr "type" "neon_ldp_q")
1473    (set_attr "fp" "yes")]
1474 )
1475
1476 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1477 ;; fairly lax checking on the second memory operation.
1478 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1479   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1480         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1481    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1482         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1483    "rtx_equal_p (XEXP (operands[2], 0),
1484                  plus_constant (Pmode,
1485                                 XEXP (operands[0], 0),
1486                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1487   "@
1488    stp\\t%w1, %w3, %0
1489    stp\\t%s1, %s3, %0"
1490   [(set_attr "type" "store_8,neon_store1_2reg")
1491    (set_attr "arch" "*,fp")]
1492 )
1493
1494 ;; Storing different modes that can still be merged
1495 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1496   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1497         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1498    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1499         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1500    "rtx_equal_p (XEXP (operands[2], 0),
1501                  plus_constant (Pmode,
1502                                 XEXP (operands[0], 0),
1503                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1504   "@
1505    stp\\t%x1, %x3, %0
1506    stp\\t%d1, %d3, %0"
1507   [(set_attr "type" "store_16,neon_store1_2reg")
1508    (set_attr "arch" "*,fp")]
1509 )
1510
1511 (define_insn "store_pair_dw_tftf"
1512   [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1513         (match_operand:TF 1 "register_operand" "w"))
1514    (set (match_operand:TF 2 "memory_operand" "=m")
1515         (match_operand:TF 3 "register_operand" "w"))]
1516    "TARGET_SIMD &&
1517     rtx_equal_p (XEXP (operands[2], 0),
1518                  plus_constant (Pmode,
1519                                 XEXP (operands[0], 0),
1520                                 GET_MODE_SIZE (TFmode)))"
1521   "stp\\t%q1, %q3, %0"
1522   [(set_attr "type" "neon_stp_q")
1523    (set_attr "fp" "yes")]
1524 )
1525
1526 ;; Load pair with post-index writeback.  This is primarily used in function
1527 ;; epilogues.
1528 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1529   [(parallel
1530     [(set (match_operand:P 0 "register_operand" "=k")
1531           (plus:P (match_operand:P 1 "register_operand" "0")
1532                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1533      (set (match_operand:GPI 2 "register_operand" "=r")
1534           (mem:GPI (match_dup 1)))
1535      (set (match_operand:GPI 3 "register_operand" "=r")
1536           (mem:GPI (plus:P (match_dup 1)
1537                    (match_operand:P 5 "const_int_operand" "n"))))])]
1538   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1539   "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1540   [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1541 )
1542
1543 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1544   [(parallel
1545     [(set (match_operand:P 0 "register_operand" "=k")
1546           (plus:P (match_operand:P 1 "register_operand" "0")
1547                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1548      (set (match_operand:GPF 2 "register_operand" "=w")
1549           (mem:GPF (match_dup 1)))
1550      (set (match_operand:GPF 3 "register_operand" "=w")
1551           (mem:GPF (plus:P (match_dup 1)
1552                    (match_operand:P 5 "const_int_operand" "n"))))])]
1553   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1554   "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1555   [(set_attr "type" "neon_load1_2reg")]
1556 )
1557
1558 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1559   [(parallel
1560     [(set (match_operand:P 0 "register_operand" "=k")
1561           (plus:P (match_operand:P 1 "register_operand" "0")
1562                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1563      (set (match_operand:TX 2 "register_operand" "=w")
1564           (mem:TX (match_dup 1)))
1565      (set (match_operand:TX 3 "register_operand" "=w")
1566           (mem:TX (plus:P (match_dup 1)
1567                           (match_operand:P 5 "const_int_operand" "n"))))])]
1568   "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1569   "ldp\\t%q2, %q3, [%1], %4"
1570   [(set_attr "type" "neon_ldp_q")]
1571 )
1572
1573 ;; Store pair with pre-index writeback.  This is primarily used in function
1574 ;; prologues.
1575 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1576   [(parallel
1577     [(set (match_operand:P 0 "register_operand" "=&k")
1578           (plus:P (match_operand:P 1 "register_operand" "0")
1579                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1580      (set (mem:GPI (plus:P (match_dup 0)
1581                    (match_dup 4)))
1582           (match_operand:GPI 2 "register_operand" "r"))
1583      (set (mem:GPI (plus:P (match_dup 0)
1584                    (match_operand:P 5 "const_int_operand" "n")))
1585           (match_operand:GPI 3 "register_operand" "r"))])]
1586   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1587   "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1588   [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1589 )
1590
1591 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1592   [(parallel
1593     [(set (match_operand:P 0 "register_operand" "=&k")
1594           (plus:P (match_operand:P 1 "register_operand" "0")
1595                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1596      (set (mem:GPF (plus:P (match_dup 0)
1597                    (match_dup 4)))
1598           (match_operand:GPF 2 "register_operand" "w"))
1599      (set (mem:GPF (plus:P (match_dup 0)
1600                    (match_operand:P 5 "const_int_operand" "n")))
1601           (match_operand:GPF 3 "register_operand" "w"))])]
1602   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1603   "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1604   [(set_attr "type" "neon_store1_2reg<q>")]
1605 )
1606
1607 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1608   [(parallel
1609     [(set (match_operand:P 0 "register_operand" "=&k")
1610           (plus:P (match_operand:P 1 "register_operand" "0")
1611                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1612      (set (mem:TX (plus:P (match_dup 0)
1613                           (match_dup 4)))
1614           (match_operand:TX 2 "register_operand" "w"))
1615      (set (mem:TX (plus:P (match_dup 0)
1616                           (match_operand:P 5 "const_int_operand" "n")))
1617           (match_operand:TX 3 "register_operand" "w"))])]
1618   "TARGET_SIMD
1619    && INTVAL (operands[5])
1620       == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1621   "stp\\t%q2, %q3, [%0, %4]!"
1622   [(set_attr "type" "neon_stp_q")]
1623 )
1624
1625 ;; -------------------------------------------------------------------
1626 ;; Sign/Zero extension
1627 ;; -------------------------------------------------------------------
1628
1629 (define_expand "<optab>sidi2"
1630   [(set (match_operand:DI 0 "register_operand")
1631         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1632   ""
1633 )
1634
1635 (define_insn "*extendsidi2_aarch64"
1636   [(set (match_operand:DI 0 "register_operand" "=r,r")
1637         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1638   ""
1639   "@
1640    sxtw\t%0, %w1
1641    ldrsw\t%0, %1"
1642   [(set_attr "type" "extend,load_4")]
1643 )
1644
1645 (define_insn "*load_pair_extendsidi2_aarch64"
1646   [(set (match_operand:DI 0 "register_operand" "=r")
1647         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1648    (set (match_operand:DI 2 "register_operand" "=r")
1649         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1650   "rtx_equal_p (XEXP (operands[3], 0),
1651                 plus_constant (Pmode,
1652                                XEXP (operands[1], 0),
1653                                GET_MODE_SIZE (SImode)))"
1654   "ldpsw\\t%0, %2, %1"
1655   [(set_attr "type" "load_8")]
1656 )
1657
1658 (define_insn "*zero_extendsidi2_aarch64"
1659   [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1660         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1661   ""
1662   "@
1663    uxtw\t%0, %w1
1664    ldr\t%w0, %1
1665    fmov\t%s0, %w1
1666    ldr\t%s0, %1
1667    fmov\t%w0, %s1
1668    fmov\t%s0, %s1"
1669   [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1670    (set_attr "arch" "*,*,fp,fp,fp,fp")]
1671 )
1672
1673 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1674   [(set (match_operand:DI 0 "register_operand" "=r,w")
1675         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1676    (set (match_operand:DI 2 "register_operand" "=r,w")
1677         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1678   "rtx_equal_p (XEXP (operands[3], 0),
1679                 plus_constant (Pmode,
1680                                XEXP (operands[1], 0),
1681                                GET_MODE_SIZE (SImode)))"
1682   "@
1683    ldp\t%w0, %w2, %1
1684    ldp\t%s0, %s2, %1"
1685   [(set_attr "type" "load_8,neon_load1_2reg")
1686    (set_attr "arch" "*,fp")]
1687 )
1688
1689 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1690   [(set (match_operand:GPI 0 "register_operand")
1691         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1692   ""
1693 )
1694
1695 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1696   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1697         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1698   ""
1699   "@
1700    sxt<SHORT:size>\t%<GPI:w>0, %w1
1701    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1702   [(set_attr "type" "extend,load_4")]
1703 )
1704
1705 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1706   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1707         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1708   ""
1709   "@
1710    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1711    ldr<SHORT:size>\t%w0, %1
1712    ldr\t%<SHORT:size>0, %1"
1713   [(set_attr "type" "logic_imm,load_4,f_loads")
1714    (set_attr "arch" "*,*,fp")]
1715 )
1716
1717 (define_expand "<optab>qihi2"
1718   [(set (match_operand:HI 0 "register_operand")
1719         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1720   ""
1721 )
1722
1723 (define_insn "*extendqihi2_aarch64"
1724   [(set (match_operand:HI 0 "register_operand" "=r,r")
1725         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1726   ""
1727   "@
1728    sxtb\t%w0, %w1
1729    ldrsb\t%w0, %1"
1730   [(set_attr "type" "extend,load_4")]
1731 )
1732
1733 (define_insn "*zero_extendqihi2_aarch64"
1734   [(set (match_operand:HI 0 "register_operand" "=r,r")
1735         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1736   ""
1737   "@
1738    and\t%w0, %w1, 255
1739    ldrb\t%w0, %1"
1740   [(set_attr "type" "logic_imm,load_4")]
1741 )
1742
1743 ;; -------------------------------------------------------------------
1744 ;; Simple arithmetic
1745 ;; -------------------------------------------------------------------
1746
1747 (define_expand "add<mode>3"
1748   [(set
1749     (match_operand:GPI 0 "register_operand")
1750     (plus:GPI (match_operand:GPI 1 "register_operand")
1751               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1752   ""
1753 {
1754   /* If operands[1] is a subreg extract the inner RTX.  */
1755   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1756
1757   /* If the constant is too large for a single instruction and isn't frame
1758      based, split off the immediate so it is available for CSE.  */
1759   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1760       && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
1761       && can_create_pseudo_p ()
1762       && (!REG_P (op1)
1763          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1764     operands[2] = force_reg (<MODE>mode, operands[2]);
1765   /* Expand polynomial additions now if the destination is the stack
1766      pointer, since we don't want to use that as a temporary.  */
1767   else if (operands[0] == stack_pointer_rtx
1768            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1769     {
1770       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1771                                 operands[2], NULL_RTX, NULL_RTX);
1772       DONE;
1773     }
1774 })
1775
1776 (define_insn "*add<mode>3_aarch64"
1777   [(set
1778     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
1779     (plus:GPI
1780      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1781      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
1782   ""
1783   "@
1784   add\\t%<w>0, %<w>1, %2
1785   add\\t%<w>0, %<w>1, %<w>2
1786   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1787   sub\\t%<w>0, %<w>1, #%n2
1788   #
1789   * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1790   * return aarch64_output_sve_addvl_addpl (operands[2]);"
1791   ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1792   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1793    (set_attr "arch" "*,*,simd,*,*,sve,sve")]
1794 )
1795
1796 ;; zero_extend version of above
1797 (define_insn "*addsi3_aarch64_uxtw"
1798   [(set
1799     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1800     (zero_extend:DI
1801      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1802               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1803   ""
1804   "@
1805   add\\t%w0, %w1, %2
1806   add\\t%w0, %w1, %w2
1807   sub\\t%w0, %w1, #%n2
1808   #"
1809   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1810 )
1811
1812 ;; If there's a free register, and we can load the constant with a
1813 ;; single instruction, do so.  This has a chance to improve scheduling.
1814 (define_peephole2
1815   [(match_scratch:GPI 3 "r")
1816    (set (match_operand:GPI 0 "register_operand")
1817         (plus:GPI
1818           (match_operand:GPI 1 "register_operand")
1819           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1820   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1821   [(set (match_dup 3) (match_dup 2))
1822    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1823 )
1824
1825 (define_peephole2
1826   [(match_scratch:SI 3 "r")
1827    (set (match_operand:DI 0 "register_operand")
1828         (zero_extend:DI
1829           (plus:SI
1830             (match_operand:SI 1 "register_operand")
1831             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1832   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1833   [(set (match_dup 3) (match_dup 2))
1834    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1835 )
1836
1837 ;; After peephole2 has had a chance to run, split any remaining long
1838 ;; additions into two add immediates.
1839 (define_split
1840   [(set (match_operand:GPI 0 "register_operand")
1841         (plus:GPI
1842           (match_operand:GPI 1 "register_operand")
1843           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1844   "epilogue_completed"
1845   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1846    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1847   {
1848     HOST_WIDE_INT i = INTVAL (operands[2]);
1849     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1850     operands[3] = GEN_INT (i - s);
1851     operands[4] = GEN_INT (s);
1852   }
1853 )
1854
1855 ;; Match addition of polynomial offsets that require one temporary, for which
1856 ;; we can use the early-clobbered destination register.  This is a separate
1857 ;; pattern so that the early clobber doesn't affect register allocation
1858 ;; for other forms of addition.  However, we still need to provide an
1859 ;; all-register alternative, in case the offset goes out of range after
1860 ;; elimination.  For completeness we might as well provide all GPR-based
1861 ;; alternatives from the main pattern.
1862 ;;
1863 ;; We don't have a pattern for additions requiring two temporaries since at
1864 ;; present LRA doesn't allow new scratches to be added during elimination.
1865 ;; Such offsets should be rare anyway.
1866 ;;
1867 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1868 ;; here would go away.  We could just handle all polynomial constants in
1869 ;; this pattern.
1870 (define_insn_and_split "*add<mode>3_poly_1"
1871   [(set
1872     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
1873     (plus:GPI
1874      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
1875      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
1876   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1877   "@
1878   add\\t%<w>0, %<w>1, %2
1879   add\\t%<w>0, %<w>1, %<w>2
1880   sub\\t%<w>0, %<w>1, #%n2
1881   #
1882   * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1883   * return aarch64_output_sve_addvl_addpl (operands[2]);
1884   #"
1885   "&& epilogue_completed
1886    && !reg_overlap_mentioned_p (operands[0], operands[1])
1887    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1888   [(const_int 0)]
1889   {
1890     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1891                               operands[2], operands[0], NULL_RTX);
1892     DONE;
1893   }
1894   ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1895   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
1896 )
1897
1898 (define_split
1899   [(set (match_operand:DI 0 "register_operand")
1900         (zero_extend:DI
1901           (plus:SI
1902             (match_operand:SI 1 "register_operand")
1903             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1904   "epilogue_completed"
1905   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1906    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1907   {
1908     HOST_WIDE_INT i = INTVAL (operands[2]);
1909     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1910     operands[3] = GEN_INT (i - s);
1911     operands[4] = GEN_INT (s);
1912     operands[5] = gen_lowpart (SImode, operands[0]);
1913   }
1914 )
1915
1916 (define_expand "addv<mode>4"
1917   [(match_operand:GPI 0 "register_operand")
1918    (match_operand:GPI 1 "register_operand")
1919    (match_operand:GPI 2 "aarch64_plus_operand")
1920    (label_ref (match_operand 3 "" ""))]
1921   ""
1922 {
1923   if (CONST_INT_P (operands[2]))
1924     emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1925                                             operands[2]));
1926   else
1927     emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1928   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1929
1930   DONE;
1931 })
1932
1933 (define_expand "uaddv<mode>4"
1934   [(match_operand:GPI 0 "register_operand")
1935    (match_operand:GPI 1 "register_operand")
1936    (match_operand:GPI 2 "register_operand")
1937    (label_ref (match_operand 3 "" ""))]
1938   ""
1939 {
1940   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1941   aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1942
1943   DONE;
1944 })
1945
1946 (define_expand "addti3"
1947   [(set (match_operand:TI 0 "register_operand")
1948         (plus:TI (match_operand:TI 1 "register_operand")
1949                  (match_operand:TI 2 "aarch64_reg_or_imm")))]
1950   ""
1951 {
1952   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1953
1954   aarch64_addti_scratch_regs (operands[1], operands[2],
1955                               &low_dest, &op1_low, &op2_low,
1956                               &high_dest, &op1_high, &op2_high);
1957
1958   if (op2_low == const0_rtx)
1959     {
1960       low_dest = op1_low;
1961       if (!aarch64_pluslong_operand (op2_high, DImode))
1962         op2_high = force_reg (DImode, op2_high);
1963       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1964     }
1965   else
1966     {
1967       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1968                                       force_reg (DImode, op2_low)));
1969       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1970                                      force_reg (DImode, op2_high)));
1971     }
1972
1973   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1974   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1975
1976   DONE;
1977 })
1978
1979 (define_expand "addvti4"
1980   [(match_operand:TI 0 "register_operand")
1981    (match_operand:TI 1 "register_operand")
1982    (match_operand:TI 2 "aarch64_reg_or_imm")
1983    (label_ref (match_operand 3 "" ""))]
1984   ""
1985 {
1986   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1987
1988   aarch64_addti_scratch_regs (operands[1], operands[2],
1989                               &low_dest, &op1_low, &op2_low,
1990                               &high_dest, &op1_high, &op2_high);
1991
1992   if (op2_low == const0_rtx)
1993     {
1994       low_dest = op1_low;
1995       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1996                                       force_reg (DImode, op2_high)));
1997     }
1998   else
1999     {
2000       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2001                                       force_reg (DImode, op2_low)));
2002       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2003                                       force_reg (DImode, op2_high)));
2004     }
2005
2006   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2007   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2008
2009   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2010   DONE;
2011 })
2012
2013 (define_expand "uaddvti4"
2014   [(match_operand:TI 0 "register_operand")
2015    (match_operand:TI 1 "register_operand")
2016    (match_operand:TI 2 "aarch64_reg_or_imm")
2017    (label_ref (match_operand 3 "" ""))]
2018   ""
2019 {
2020   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2021
2022   aarch64_addti_scratch_regs (operands[1], operands[2],
2023                               &low_dest, &op1_low, &op2_low,
2024                               &high_dest, &op1_high, &op2_high);
2025
2026   if (op2_low == const0_rtx)
2027     {
2028       low_dest = op1_low;
2029       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2030                                       force_reg (DImode, op2_high)));
2031     }
2032   else
2033     {
2034       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2035                                       force_reg (DImode, op2_low)));
2036       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2037                                       force_reg (DImode, op2_high)));
2038     }
2039
2040   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2041   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2042
2043   aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2044   DONE;
2045  })
2046
2047 (define_insn "add<mode>3_compare0"
2048   [(set (reg:CC_NZ CC_REGNUM)
2049         (compare:CC_NZ
2050          (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2051                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2052          (const_int 0)))
2053    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2054         (plus:GPI (match_dup 1) (match_dup 2)))]
2055   ""
2056   "@
2057   adds\\t%<w>0, %<w>1, %<w>2
2058   adds\\t%<w>0, %<w>1, %2
2059   subs\\t%<w>0, %<w>1, #%n2"
2060   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2061 )
2062
2063 ;; zero_extend version of above
2064 (define_insn "*addsi3_compare0_uxtw"
2065   [(set (reg:CC_NZ CC_REGNUM)
2066         (compare:CC_NZ
2067          (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2068                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2069          (const_int 0)))
2070    (set (match_operand:DI 0 "register_operand" "=r,r,r")
2071         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2072   ""
2073   "@
2074   adds\\t%w0, %w1, %w2
2075   adds\\t%w0, %w1, %2
2076   subs\\t%w0, %w1, #%n2"
2077   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2078 )
2079
2080 (define_insn "*add<mode>3_compareC_cconly"
2081   [(set (reg:CC_C CC_REGNUM)
2082         (compare:CC_C
2083           (plus:GPI
2084             (match_operand:GPI 0 "register_operand" "r,r,r")
2085             (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2086           (match_dup 0)))]
2087   ""
2088   "@
2089   cmn\\t%<w>0, %<w>1
2090   cmn\\t%<w>0, %1
2091   cmp\\t%<w>0, #%n1"
2092   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2093 )
2094
2095 (define_insn "add<mode>3_compareC"
2096   [(set (reg:CC_C CC_REGNUM)
2097         (compare:CC_C
2098           (plus:GPI
2099             (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2100             (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2101           (match_dup 1)))
2102    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2103         (plus:GPI (match_dup 1) (match_dup 2)))]
2104   ""
2105   "@
2106   adds\\t%<w>0, %<w>1, %<w>2
2107   adds\\t%<w>0, %<w>1, %2
2108   subs\\t%<w>0, %<w>1, #%n2"
2109   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2110 )
2111
2112 (define_insn "*add<mode>3_compareV_cconly_imm"
2113   [(set (reg:CC_V CC_REGNUM)
2114         (compare:CC_V
2115           (plus:<DWI>
2116             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2117             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2118           (sign_extend:<DWI>
2119            (plus:GPI
2120             (match_dup 0)
2121             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2122   "INTVAL (operands[1]) == INTVAL (operands[2])"
2123   "@
2124   cmn\\t%<w>0, %<w>1
2125   cmp\\t%<w>0, #%n1"
2126   [(set_attr "type" "alus_imm")]
2127 )
2128
2129 (define_insn "*add<mode>3_compareV_cconly"
2130   [(set (reg:CC_V CC_REGNUM)
2131         (compare:CC_V
2132           (plus:<DWI>
2133             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2134             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2135           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2136   ""
2137   "cmn\\t%<w>0, %<w>1"
2138   [(set_attr "type" "alus_sreg")]
2139 )
2140
2141 (define_insn "add<mode>3_compareV_imm"
2142   [(set (reg:CC_V CC_REGNUM)
2143         (compare:CC_V
2144           (plus:<DWI>
2145             (sign_extend:<DWI>
2146               (match_operand:GPI 1 "register_operand" "rk,rk"))
2147             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2148           (sign_extend:<DWI>
2149             (plus:GPI (match_dup 1) (match_dup 2)))))
2150    (set (match_operand:GPI 0 "register_operand" "=r,r")
2151         (plus:GPI (match_dup 1) (match_dup 2)))]
2152    ""
2153    "@
2154    adds\\t%<w>0, %<w>1, %<w>2
2155    subs\\t%<w>0, %<w>1, #%n2"
2156   [(set_attr "type" "alus_imm,alus_imm")]
2157 )
2158
2159 (define_insn "add<mode>3_compareV"
2160   [(set (reg:CC_V CC_REGNUM)
2161         (compare:CC_V
2162           (plus:<DWI>
2163             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2164             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2165           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2166    (set (match_operand:GPI 0 "register_operand" "=r")
2167         (plus:GPI (match_dup 1) (match_dup 2)))]
2168   ""
2169   "adds\\t%<w>0, %<w>1, %<w>2"
2170   [(set_attr "type" "alus_sreg")]
2171 )
2172
2173 (define_insn "*adds_shift_imm_<mode>"
2174   [(set (reg:CC_NZ CC_REGNUM)
2175         (compare:CC_NZ
2176          (plus:GPI (ASHIFT:GPI
2177                     (match_operand:GPI 1 "register_operand" "r")
2178                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2179                    (match_operand:GPI 3 "register_operand" "r"))
2180          (const_int 0)))
2181    (set (match_operand:GPI 0 "register_operand" "=r")
2182         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2183                   (match_dup 3)))]
2184   ""
2185   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2186   [(set_attr "type" "alus_shift_imm")]
2187 )
2188
2189 (define_insn "*subs_shift_imm_<mode>"
2190   [(set (reg:CC_NZ CC_REGNUM)
2191         (compare:CC_NZ
2192          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2193                     (ASHIFT:GPI
2194                      (match_operand:GPI 2 "register_operand" "r")
2195                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2196          (const_int 0)))
2197    (set (match_operand:GPI 0 "register_operand" "=r")
2198         (minus:GPI (match_dup 1)
2199                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2200   ""
2201   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2202   [(set_attr "type" "alus_shift_imm")]
2203 )
2204
2205 (define_insn "*adds_mul_imm_<mode>"
2206   [(set (reg:CC_NZ CC_REGNUM)
2207         (compare:CC_NZ
2208          (plus:GPI (mult:GPI
2209                     (match_operand:GPI 1 "register_operand" "r")
2210                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2211                    (match_operand:GPI 3 "register_operand" "r"))
2212          (const_int 0)))
2213    (set (match_operand:GPI 0 "register_operand" "=r")
2214         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2215                   (match_dup 3)))]
2216   ""
2217   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2218   [(set_attr "type" "alus_shift_imm")]
2219 )
2220
2221 (define_insn "*subs_mul_imm_<mode>"
2222   [(set (reg:CC_NZ CC_REGNUM)
2223         (compare:CC_NZ
2224          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2225                     (mult:GPI
2226                      (match_operand:GPI 2 "register_operand" "r")
2227                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2228          (const_int 0)))
2229    (set (match_operand:GPI 0 "register_operand" "=r")
2230         (minus:GPI (match_dup 1)
2231                    (mult:GPI (match_dup 2) (match_dup 3))))]
2232   ""
2233   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2234   [(set_attr "type" "alus_shift_imm")]
2235 )
2236
2237 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2238   [(set (reg:CC_NZ CC_REGNUM)
2239         (compare:CC_NZ
2240          (plus:GPI
2241           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2242           (match_operand:GPI 2 "register_operand" "rk"))
2243         (const_int 0)))
2244    (set (match_operand:GPI 0 "register_operand" "=r")
2245         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2246   ""
2247   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2248   [(set_attr "type" "alus_ext")]
2249 )
2250
2251 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2252   [(set (reg:CC_NZ CC_REGNUM)
2253         (compare:CC_NZ
2254          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2255                     (ANY_EXTEND:GPI
2256                      (match_operand:ALLX 2 "register_operand" "r")))
2257         (const_int 0)))
2258    (set (match_operand:GPI 0 "register_operand" "=r")
2259         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2260   ""
2261   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2262   [(set_attr "type" "alus_ext")]
2263 )
2264
2265 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2266   [(set (reg:CC_NZ CC_REGNUM)
2267         (compare:CC_NZ
2268          (plus:GPI (ashift:GPI
2269                     (ANY_EXTEND:GPI
2270                      (match_operand:ALLX 1 "register_operand" "r"))
2271                     (match_operand 2 "aarch64_imm3" "Ui3"))
2272                    (match_operand:GPI 3 "register_operand" "rk"))
2273          (const_int 0)))
2274    (set (match_operand:GPI 0 "register_operand" "=rk")
2275         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2276                               (match_dup 2))
2277                   (match_dup 3)))]
2278   ""
2279   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2280   [(set_attr "type" "alus_ext")]
2281 )
2282
2283 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2284   [(set (reg:CC_NZ CC_REGNUM)
2285         (compare:CC_NZ
2286          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2287                     (ashift:GPI
2288                      (ANY_EXTEND:GPI
2289                       (match_operand:ALLX 2 "register_operand" "r"))
2290                      (match_operand 3 "aarch64_imm3" "Ui3")))
2291          (const_int 0)))
2292    (set (match_operand:GPI 0 "register_operand" "=rk")
2293         (minus:GPI (match_dup 1)
2294                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2295                                (match_dup 3))))]
2296   ""
2297   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2298   [(set_attr "type" "alus_ext")]
2299 )
2300
2301 (define_insn "*adds_<optab><mode>_multp2"
2302   [(set (reg:CC_NZ CC_REGNUM)
2303         (compare:CC_NZ
2304          (plus:GPI (ANY_EXTRACT:GPI
2305                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2306                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2307                     (match_operand 3 "const_int_operand" "n")
2308                     (const_int 0))
2309                    (match_operand:GPI 4 "register_operand" "rk"))
2310         (const_int 0)))
2311    (set (match_operand:GPI 0 "register_operand" "=r")
2312         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2313                                    (match_dup 3)
2314                                    (const_int 0))
2315                   (match_dup 4)))]
2316   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2317   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2318   [(set_attr "type" "alus_ext")]
2319 )
2320
2321 (define_insn "*subs_<optab><mode>_multp2"
2322   [(set (reg:CC_NZ CC_REGNUM)
2323         (compare:CC_NZ
2324          (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2325                     (ANY_EXTRACT:GPI
2326                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2327                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2328                      (match_operand 3 "const_int_operand" "n")
2329                      (const_int 0)))
2330         (const_int 0)))
2331    (set (match_operand:GPI 0 "register_operand" "=r")
2332         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2333                                   (mult:GPI (match_dup 1) (match_dup 2))
2334                                   (match_dup 3)
2335                                   (const_int 0))))]
2336   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2337   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2338   [(set_attr "type" "alus_ext")]
2339 )
2340
2341 (define_insn "*add<mode>3nr_compare0"
2342   [(set (reg:CC_NZ CC_REGNUM)
2343         (compare:CC_NZ
2344          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2345                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2346          (const_int 0)))]
2347   ""
2348   "@
2349   cmn\\t%<w>0, %<w>1
2350   cmn\\t%<w>0, %1
2351   cmp\\t%<w>0, #%n1"
2352   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2353 )
2354
2355 (define_insn "aarch64_sub<mode>_compare0"
2356   [(set (reg:CC_NZ CC_REGNUM)
2357         (compare:CC_NZ
2358          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2359                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2360          (const_int 0)))]
2361   ""
2362   "cmp\\t%<w>0, %<w>1"
2363   [(set_attr "type" "alus_sreg")]
2364 )
2365
2366 (define_insn "*compare_neg<mode>"
2367   [(set (reg:CC_Z CC_REGNUM)
2368         (compare:CC_Z
2369          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2370          (match_operand:GPI 1 "register_operand" "r")))]
2371   ""
2372   "cmn\\t%<w>1, %<w>0"
2373   [(set_attr "type" "alus_sreg")]
2374 )
2375
2376 (define_insn "*add_<shift>_<mode>"
2377   [(set (match_operand:GPI 0 "register_operand" "=r")
2378         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2379                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2380                   (match_operand:GPI 3 "register_operand" "r")))]
2381   ""
2382   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2383   [(set_attr "type" "alu_shift_imm")]
2384 )
2385
2386 ;; zero_extend version of above
2387 (define_insn "*add_<shift>_si_uxtw"
2388   [(set (match_operand:DI 0 "register_operand" "=r")
2389         (zero_extend:DI
2390          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2391                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2392                   (match_operand:SI 3 "register_operand" "r"))))]
2393   ""
2394   "add\\t%w0, %w3, %w1, <shift> %2"
2395   [(set_attr "type" "alu_shift_imm")]
2396 )
2397
2398 (define_insn "*add_mul_imm_<mode>"
2399   [(set (match_operand:GPI 0 "register_operand" "=r")
2400         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2401                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2402                   (match_operand:GPI 3 "register_operand" "r")))]
2403   ""
2404   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2405   [(set_attr "type" "alu_shift_imm")]
2406 )
2407
2408 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2409   [(set (match_operand:GPI 0 "register_operand" "=rk")
2410         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2411                   (match_operand:GPI 2 "register_operand" "r")))]
2412   ""
2413   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2414   [(set_attr "type" "alu_ext")]
2415 )
2416
2417 ;; zero_extend version of above
2418 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2419   [(set (match_operand:DI 0 "register_operand" "=rk")
2420         (zero_extend:DI
2421          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2422                   (match_operand:GPI 2 "register_operand" "r"))))]
2423   ""
2424   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2425   [(set_attr "type" "alu_ext")]
2426 )
2427
2428 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2429   [(set (match_operand:GPI 0 "register_operand" "=rk")
2430         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2431                                (match_operand:ALLX 1 "register_operand" "r"))
2432                               (match_operand 2 "aarch64_imm3" "Ui3"))
2433                   (match_operand:GPI 3 "register_operand" "r")))]
2434   ""
2435   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2436   [(set_attr "type" "alu_ext")]
2437 )
2438
2439 ;; zero_extend version of above
2440 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2441   [(set (match_operand:DI 0 "register_operand" "=rk")
2442         (zero_extend:DI
2443          (plus:SI (ashift:SI (ANY_EXTEND:SI
2444                               (match_operand:SHORT 1 "register_operand" "r"))
2445                              (match_operand 2 "aarch64_imm3" "Ui3"))
2446                   (match_operand:SI 3 "register_operand" "r"))))]
2447   ""
2448   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2449   [(set_attr "type" "alu_ext")]
2450 )
2451
2452 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2453   [(set (match_operand:GPI 0 "register_operand" "=rk")
2454         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2455                              (match_operand:ALLX 1 "register_operand" "r"))
2456                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2457                   (match_operand:GPI 3 "register_operand" "r")))]
2458   ""
2459   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2460   [(set_attr "type" "alu_ext")]
2461 )
2462
2463 ;; zero_extend version of above
2464 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2465   [(set (match_operand:DI 0 "register_operand" "=rk")
2466         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2467                              (match_operand:SHORT 1 "register_operand" "r"))
2468                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2469                   (match_operand:SI 3 "register_operand" "r"))))]
2470   ""
2471   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2472   [(set_attr "type" "alu_ext")]
2473 )
2474
2475 (define_insn "*add_<optab><mode>_multp2"
2476   [(set (match_operand:GPI 0 "register_operand" "=rk")
2477         (plus:GPI (ANY_EXTRACT:GPI
2478                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2479                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2480                    (match_operand 3 "const_int_operand" "n")
2481                    (const_int 0))
2482                   (match_operand:GPI 4 "register_operand" "r")))]
2483   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2484   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2485   [(set_attr "type" "alu_ext")]
2486 )
2487
2488 ;; zero_extend version of above
2489 (define_insn "*add_<optab>si_multp2_uxtw"
2490   [(set (match_operand:DI 0 "register_operand" "=rk")
2491         (zero_extend:DI
2492          (plus:SI (ANY_EXTRACT:SI
2493                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2494                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2495                    (match_operand 3 "const_int_operand" "n")
2496                    (const_int 0))
2497                   (match_operand:SI 4 "register_operand" "r"))))]
2498   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2499   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2500   [(set_attr "type" "alu_ext")]
2501 )
2502
2503 (define_expand "add<mode>3_carryin"
2504   [(set (match_operand:GPI 0 "register_operand")
2505         (plus:GPI
2506           (plus:GPI
2507             (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2508             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2509           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2510    ""
2511    ""
2512 )
2513
2514 ;; Note that add with carry with two zero inputs is matched by cset,
2515 ;; and that add with carry with one zero input is matched by cinc.
2516
2517 (define_insn "*add<mode>3_carryin"
2518   [(set (match_operand:GPI 0 "register_operand" "=r")
2519         (plus:GPI
2520           (plus:GPI
2521             (match_operand:GPI 3 "aarch64_carry_operation" "")
2522             (match_operand:GPI 1 "register_operand" "r"))
2523           (match_operand:GPI 2 "register_operand" "r")))]
2524    ""
2525    "adc\\t%<w>0, %<w>1, %<w>2"
2526   [(set_attr "type" "adc_reg")]
2527 )
2528
2529 ;; zero_extend version of above
2530 (define_insn "*addsi3_carryin_uxtw"
2531   [(set (match_operand:DI 0 "register_operand" "=r")
2532         (zero_extend:DI
2533           (plus:SI
2534             (plus:SI
2535               (match_operand:SI 3 "aarch64_carry_operation" "")
2536               (match_operand:SI 1 "register_operand" "r"))
2537             (match_operand:SI 2 "register_operand" "r"))))]
2538    ""
2539    "adc\\t%w0, %w1, %w2"
2540   [(set_attr "type" "adc_reg")]
2541 )
2542
2543 (define_expand "add<mode>3_carryinC"
2544   [(parallel
2545      [(set (match_dup 3)
2546            (compare:CC_ADC
2547              (plus:<DWI>
2548                (plus:<DWI>
2549                  (match_dup 4)
2550                  (zero_extend:<DWI>
2551                    (match_operand:GPI 1 "register_operand")))
2552                (zero_extend:<DWI>
2553                  (match_operand:GPI 2 "register_operand")))
2554              (match_dup 6)))
2555       (set (match_operand:GPI 0 "register_operand")
2556            (plus:GPI
2557              (plus:GPI (match_dup 5) (match_dup 1))
2558              (match_dup 2)))])]
2559    ""
2560 {
2561   operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2562   rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2563   operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2564   operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2565   operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2566                                       << GET_MODE_BITSIZE (<MODE>mode),
2567                                       TImode);
2568 })
2569
2570 (define_insn "*add<mode>3_carryinC_zero"
2571   [(set (reg:CC_ADC CC_REGNUM)
2572         (compare:CC_ADC
2573           (plus:<DWI>
2574             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2575             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2576           (match_operand 4 "const_scalar_int_operand" "")))
2577    (set (match_operand:GPI 0 "register_operand" "=r")
2578         (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2579                   (match_dup 1)))]
2580   "rtx_mode_t (operands[4], <DWI>mode)
2581    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2582    "adcs\\t%<w>0, %<w>1, <w>zr"
2583   [(set_attr "type" "adc_reg")]
2584 )
2585
2586 (define_insn "*add<mode>3_carryinC"
2587   [(set (reg:CC_ADC CC_REGNUM)
2588         (compare:CC_ADC
2589           (plus:<DWI>
2590             (plus:<DWI>
2591               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2592               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2593             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2594           (match_operand 5 "const_scalar_int_operand" "")))
2595    (set (match_operand:GPI 0 "register_operand" "=r")
2596         (plus:GPI
2597           (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2598                     (match_dup 1))
2599           (match_dup 2)))]
2600   "rtx_mode_t (operands[5], <DWI>mode)
2601    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2602    "adcs\\t%<w>0, %<w>1, %<w>2"
2603   [(set_attr "type" "adc_reg")]
2604 )
2605
2606 (define_expand "add<mode>3_carryinV"
2607   [(parallel
2608      [(set (reg:CC_V CC_REGNUM)
2609            (compare:CC_V
2610              (plus:<DWI>
2611                (plus:<DWI>
2612                  (match_dup 3)
2613                  (sign_extend:<DWI>
2614                    (match_operand:GPI 1 "register_operand")))
2615                (sign_extend:<DWI>
2616                  (match_operand:GPI 2 "register_operand")))
2617            (sign_extend:<DWI>
2618              (plus:GPI
2619                (plus:GPI (match_dup 4) (match_dup 1))
2620                (match_dup 2)))))
2621       (set (match_operand:GPI 0 "register_operand")
2622            (plus:GPI
2623              (plus:GPI (match_dup 4) (match_dup 1))
2624              (match_dup 2)))])]
2625    ""
2626 {
2627   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2628   operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2629   operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2630 })
2631
2632 (define_insn "*add<mode>3_carryinV_zero"
2633   [(set (reg:CC_V CC_REGNUM)
2634         (compare:CC_V
2635           (plus:<DWI>
2636             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2637             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2638           (sign_extend:<DWI>
2639             (plus:GPI
2640               (match_operand:GPI 3 "aarch64_carry_operation" "")
2641               (match_dup 1)))))
2642    (set (match_operand:GPI 0 "register_operand" "=r")
2643         (plus:GPI (match_dup 3) (match_dup 1)))]
2644    ""
2645    "adcs\\t%<w>0, %<w>1, <w>zr"
2646   [(set_attr "type" "adc_reg")]
2647 )
2648
2649 (define_insn "*add<mode>3_carryinV"
2650   [(set (reg:CC_V CC_REGNUM)
2651         (compare:CC_V
2652           (plus:<DWI>
2653             (plus:<DWI>
2654               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2655               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2656             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2657           (sign_extend:<DWI>
2658             (plus:GPI
2659               (plus:GPI
2660                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2661                 (match_dup 1))
2662               (match_dup 2)))))
2663    (set (match_operand:GPI 0 "register_operand" "=r")
2664         (plus:GPI
2665           (plus:GPI (match_dup 4) (match_dup 1))
2666           (match_dup 2)))]
2667    ""
2668    "adcs\\t%<w>0, %<w>1, %<w>2"
2669   [(set_attr "type" "adc_reg")]
2670 )
2671
2672 (define_insn "*add_uxt<mode>_shift2"
2673   [(set (match_operand:GPI 0 "register_operand" "=rk")
2674         (plus:GPI (and:GPI
2675                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2676                                (match_operand 2 "aarch64_imm3" "Ui3"))
2677                    (match_operand 3 "const_int_operand" "n"))
2678                   (match_operand:GPI 4 "register_operand" "r")))]
2679   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2680   "*
2681   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2682                                            INTVAL (operands[3])));
2683   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2684   [(set_attr "type" "alu_ext")]
2685 )
2686
2687 ;; zero_extend version of above
2688 (define_insn "*add_uxtsi_shift2_uxtw"
2689   [(set (match_operand:DI 0 "register_operand" "=rk")
2690         (zero_extend:DI
2691          (plus:SI (and:SI
2692                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2693                               (match_operand 2 "aarch64_imm3" "Ui3"))
2694                    (match_operand 3 "const_int_operand" "n"))
2695                   (match_operand:SI 4 "register_operand" "r"))))]
2696   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2697   "*
2698   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2699                                            INTVAL (operands[3])));
2700   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2701   [(set_attr "type" "alu_ext")]
2702 )
2703
2704 (define_insn "*add_uxt<mode>_multp2"
2705   [(set (match_operand:GPI 0 "register_operand" "=rk")
2706         (plus:GPI (and:GPI
2707                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2708                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2709                    (match_operand 3 "const_int_operand" "n"))
2710                   (match_operand:GPI 4 "register_operand" "r")))]
2711   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2712   "*
2713   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2714                                            INTVAL (operands[3])));
2715   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2716   [(set_attr "type" "alu_ext")]
2717 )
2718
2719 ;; zero_extend version of above
2720 (define_insn "*add_uxtsi_multp2_uxtw"
2721   [(set (match_operand:DI 0 "register_operand" "=rk")
2722         (zero_extend:DI
2723          (plus:SI (and:SI
2724                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2725                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2726                    (match_operand 3 "const_int_operand" "n"))
2727                   (match_operand:SI 4 "register_operand" "r"))))]
2728   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2729   "*
2730   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2731                                            INTVAL (operands[3])));
2732   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2733   [(set_attr "type" "alu_ext")]
2734 )
2735
2736 (define_insn "subsi3"
2737   [(set (match_operand:SI 0 "register_operand" "=rk")
2738         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2739                   (match_operand:SI 2 "register_operand" "r")))]
2740   ""
2741   "sub\\t%w0, %w1, %w2"
2742   [(set_attr "type" "alu_sreg")]
2743 )
2744
2745 ;; zero_extend version of above
2746 (define_insn "*subsi3_uxtw"
2747   [(set (match_operand:DI 0 "register_operand" "=rk")
2748         (zero_extend:DI
2749          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2750                    (match_operand:SI 2 "register_operand" "r"))))]
2751   ""
2752   "sub\\t%w0, %w1, %w2"
2753   [(set_attr "type" "alu_sreg")]
2754 )
2755
2756 (define_insn "subdi3"
2757   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2758         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2759                   (match_operand:DI 2 "register_operand" "r,w")))]
2760   ""
2761   "@
2762    sub\\t%x0, %x1, %x2
2763    sub\\t%d0, %d1, %d2"
2764   [(set_attr "type" "alu_sreg, neon_sub")
2765    (set_attr "arch" "*,simd")]
2766 )
2767
2768 (define_expand "subv<GPI:mode>4"
2769   [(match_operand:GPI 0 "register_operand")
2770    (match_operand:GPI 1 "register_operand")
2771    (match_operand:GPI 2 "aarch64_plus_operand")
2772    (label_ref (match_operand 3 "" ""))]
2773   ""
2774 {
2775   if (CONST_INT_P (operands[2]))
2776     emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2777   else
2778     emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2779   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2780
2781   DONE;
2782 })
2783
2784 (define_insn "subv<GPI:mode>_insn"
2785   [(set (reg:CC_V CC_REGNUM)
2786         (compare:CC_V
2787          (sign_extend:<DWI>
2788           (minus:GPI
2789            (match_operand:GPI 1 "register_operand" "rk")
2790            (match_operand:GPI 2 "register_operand" "r")))
2791          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2792                       (sign_extend:<DWI> (match_dup 2)))))
2793    (set (match_operand:GPI 0 "register_operand" "=r")
2794         (minus:GPI (match_dup 1) (match_dup 2)))]
2795   ""
2796   "subs\\t%<w>0, %<w>1, %<w>2"
2797   [(set_attr "type" "alus_sreg")]
2798 )
2799
2800 (define_insn "subv<GPI:mode>_imm"
2801   [(set (reg:CC_V CC_REGNUM)
2802         (compare:CC_V
2803          (sign_extend:<DWI>
2804           (minus:GPI
2805            (match_operand:GPI 1 "register_operand" "rk,rk")
2806            (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2807          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2808                       (match_dup 2))))
2809    (set (match_operand:GPI 0 "register_operand" "=r,r")
2810         (minus:GPI (match_dup 1) (match_dup 2)))]
2811   ""
2812   "@
2813    subs\\t%<w>0, %<w>1, %2
2814    adds\\t%<w>0, %<w>1, #%n2"
2815   [(set_attr "type" "alus_sreg")]
2816 )
2817
2818 (define_expand "negv<GPI:mode>3"
2819   [(match_operand:GPI 0 "register_operand")
2820    (match_operand:GPI 1 "register_operand")
2821    (label_ref (match_operand 2 "" ""))]
2822   ""
2823   {
2824     emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2825     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2826
2827     DONE;
2828   }
2829 )
2830
2831 (define_insn "negv<GPI:mode>_insn"
2832   [(set (reg:CC_V CC_REGNUM)
2833         (compare:CC_V
2834          (sign_extend:<DWI>
2835           (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2836          (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2837    (set (match_operand:GPI 0 "register_operand" "=r")
2838         (neg:GPI (match_dup 1)))]
2839   ""
2840   "negs\\t%<w>0, %<w>1"
2841   [(set_attr "type" "alus_sreg")]
2842 )
2843
2844 (define_insn "negv<GPI:mode>_cmp_only"
2845   [(set (reg:CC_V CC_REGNUM)
2846         (compare:CC_V
2847          (sign_extend:<DWI>
2848           (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2849          (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2850   ""
2851   "negs\\t%<w>zr, %<w>0"
2852   [(set_attr "type" "alus_sreg")]
2853 )
2854
2855 (define_insn "*cmpv<GPI:mode>_insn"
2856   [(set (reg:CC_V CC_REGNUM)
2857         (compare:CC_V
2858          (sign_extend:<DWI>
2859           (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2860                      (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2861          (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2862                     (sign_extend:<DWI> (match_dup 1)))))]
2863   ""
2864   "@
2865    cmp\\t%<w>0, %<w>1
2866    cmp\\t%<w>0, %1
2867    cmp\\t%<w>0, #%n1"
2868   [(set_attr "type" "alus_sreg")]
2869 )
2870
2871 (define_expand "usubv<mode>4"
2872   [(match_operand:GPI 0 "register_operand")
2873    (match_operand:GPI 1 "aarch64_reg_or_zero")
2874    (match_operand:GPI 2 "aarch64_reg_or_zero")
2875    (label_ref (match_operand 3 "" ""))]
2876   ""
2877 {
2878   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2879   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2880
2881   DONE;
2882 })
2883
2884 (define_expand "subti3"
2885   [(set (match_operand:TI 0 "register_operand")
2886         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2887                   (match_operand:TI 2 "register_operand")))]
2888   ""
2889 {
2890   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2891
2892   aarch64_subvti_scratch_regs (operands[1], operands[2],
2893                                &low_dest, &op1_low, &op2_low,
2894                                &high_dest, &op1_high, &op2_high);
2895
2896   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2897   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2898
2899   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2900   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2901   DONE;
2902 })
2903
2904 (define_expand "subvti4"
2905   [(match_operand:TI 0 "register_operand")
2906    (match_operand:TI 1 "register_operand")
2907    (match_operand:TI 2 "aarch64_reg_or_imm")
2908    (label_ref (match_operand 3 "" ""))]
2909   ""
2910 {
2911   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2912
2913   aarch64_subvti_scratch_regs (operands[1], operands[2],
2914                                &low_dest, &op1_low, &op2_low,
2915                                &high_dest, &op1_high, &op2_high);
2916   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2917                          high_dest, op1_high, op2_high, false);
2918
2919   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2920   DONE;
2921 })
2922
2923 (define_expand "usubvti4"
2924   [(match_operand:TI 0 "register_operand")
2925    (match_operand:TI 1 "register_operand")
2926    (match_operand:TI 2 "aarch64_reg_or_imm")
2927    (label_ref (match_operand 3 "" ""))]
2928   ""
2929 {
2930   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2931
2932   aarch64_subvti_scratch_regs (operands[1], operands[2],
2933                                     &low_dest, &op1_low, &op2_low,
2934                                &high_dest, &op1_high, &op2_high);
2935   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2936                          high_dest, op1_high, op2_high, true);
2937
2938   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2939   DONE;
2940 })
2941
2942 (define_expand "negvti3"
2943   [(match_operand:TI 0 "register_operand")
2944    (match_operand:TI 1 "register_operand")
2945    (label_ref (match_operand 2 "" ""))]
2946   ""
2947   {
2948     emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2949                                    gen_lowpart (DImode, operands[1])));
2950     emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2951                                     gen_highpart (DImode, operands[1])));
2952     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2953
2954     DONE;
2955   }
2956 )
2957
2958 (define_insn "negdi_carryout"
2959   [(set (reg:CC CC_REGNUM)
2960         (compare:CC
2961          (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2962    (set (match_operand:DI 0 "register_operand" "=r")
2963         (neg:DI (match_dup 1)))]
2964   ""
2965   "negs\\t%0, %1"
2966   [(set_attr "type" "alus_sreg")]
2967 )
2968
2969 (define_insn "negvdi_carryinV"
2970   [(set (reg:CC_V CC_REGNUM)
2971         (compare:CC_V
2972          (neg:TI (plus:TI
2973                   (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2974                   (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2975          (sign_extend:TI
2976           (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2977                            (match_dup 1))))))
2978    (set (match_operand:DI 0 "register_operand" "=r")
2979         (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2980                          (match_dup 1))))]
2981   ""
2982   "ngcs\\t%0, %1"
2983   [(set_attr "type" "alus_sreg")]
2984 )
2985
2986 (define_insn "*sub<mode>3_compare0"
2987   [(set (reg:CC_NZ CC_REGNUM)
2988         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2989                                   (match_operand:GPI 2 "register_operand" "r"))
2990                        (const_int 0)))
2991    (set (match_operand:GPI 0 "register_operand" "=r")
2992         (minus:GPI (match_dup 1) (match_dup 2)))]
2993   ""
2994   "subs\\t%<w>0, %<w>1, %<w>2"
2995   [(set_attr "type" "alus_sreg")]
2996 )
2997
2998 ;; zero_extend version of above
2999 (define_insn "*subsi3_compare0_uxtw"
3000   [(set (reg:CC_NZ CC_REGNUM)
3001         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3002                                  (match_operand:SI 2 "register_operand" "r"))
3003                        (const_int 0)))
3004    (set (match_operand:DI 0 "register_operand" "=r")
3005         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3006   ""
3007   "subs\\t%w0, %w1, %w2"
3008   [(set_attr "type" "alus_sreg")]
3009 )
3010
3011 (define_insn "sub<mode>3_compare1_imm"
3012   [(set (reg:CC CC_REGNUM)
3013         (compare:CC
3014           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
3015           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3016    (set (match_operand:GPI 0 "register_operand" "=r,r")
3017         (plus:GPI
3018           (match_dup 1)
3019           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3020   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3021   "@
3022   subs\\t%<w>0, %<w>1, %2
3023   adds\\t%<w>0, %<w>1, #%n2"
3024   [(set_attr "type" "alus_imm")]
3025 )
3026
3027 (define_insn "sub<mode>3_compare1"
3028   [(set (reg:CC CC_REGNUM)
3029         (compare:CC
3030           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3031           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3032    (set (match_operand:GPI 0 "register_operand" "=r")
3033         (minus:GPI (match_dup 1) (match_dup 2)))]
3034   ""
3035   "subs\\t%<w>0, %<w>1, %<w>2"
3036   [(set_attr "type" "alus_sreg")]
3037 )
3038
3039 (define_peephole2
3040   [(set (match_operand:GPI 0 "aarch64_general_reg")
3041         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3042                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
3043    (set (reg:CC CC_REGNUM)
3044         (compare:CC
3045           (match_dup 1)
3046           (match_dup 2)))]
3047   "!reg_overlap_mentioned_p (operands[0], operands[1])
3048    && !reg_overlap_mentioned_p (operands[0], operands[2])"
3049   [(const_int 0)]
3050   {
3051     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3052                                          operands[2]));
3053     DONE;
3054   }
3055 )
3056
3057 ;; Same as the above peephole but with the compare and minus in
3058 ;; swapped order.  The restriction on overlap between operand 0
3059 ;; and operands 1 and 2 doesn't apply here.
3060 (define_peephole2
3061   [(set (reg:CC CC_REGNUM)
3062         (compare:CC
3063           (match_operand:GPI 1 "aarch64_reg_or_zero")
3064           (match_operand:GPI 2 "aarch64_reg_or_zero")))
3065    (set (match_operand:GPI 0 "aarch64_general_reg")
3066         (minus:GPI (match_dup 1)
3067                    (match_dup 2)))]
3068   ""
3069   [(const_int 0)]
3070   {
3071     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3072                                          operands[2]));
3073     DONE;
3074   }
3075 )
3076
3077 (define_peephole2
3078   [(set (match_operand:GPI 0 "aarch64_general_reg")
3079         (plus:GPI (match_operand:GPI 1 "register_operand")
3080                   (match_operand:GPI 2 "aarch64_plus_immediate")))
3081    (set (reg:CC CC_REGNUM)
3082         (compare:CC
3083           (match_dup 1)
3084           (match_operand:GPI 3 "const_int_operand")))]
3085   "!reg_overlap_mentioned_p (operands[0], operands[1])
3086    && INTVAL (operands[3]) == -INTVAL (operands[2])"
3087   [(const_int 0)]
3088   {
3089     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3090                                          operands[3], operands[2]));
3091     DONE;
3092   }
3093 )
3094
3095 ;; Same as the above peephole but with the compare and minus in
3096 ;; swapped order.  The restriction on overlap between operand 0
3097 ;; and operands 1 doesn't apply here.
3098 (define_peephole2
3099   [(set (reg:CC CC_REGNUM)
3100         (compare:CC
3101           (match_operand:GPI 1 "register_operand")
3102           (match_operand:GPI 3 "const_int_operand")))
3103    (set (match_operand:GPI 0 "aarch64_general_reg")
3104         (plus:GPI (match_dup 1)
3105                   (match_operand:GPI 2 "aarch64_plus_immediate")))]
3106   "INTVAL (operands[3]) == -INTVAL (operands[2])"
3107   [(const_int 0)]
3108   {
3109     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3110                                          operands[3], operands[2]));
3111     DONE;
3112   }
3113 )
3114
3115 (define_insn "*sub_<shift>_<mode>"
3116   [(set (match_operand:GPI 0 "register_operand" "=r")
3117         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3118                    (ASHIFT:GPI
3119                     (match_operand:GPI 1 "register_operand" "r")
3120                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3121   ""
3122   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3123   [(set_attr "type" "alu_shift_imm")]
3124 )
3125
3126 ;; zero_extend version of above
3127 (define_insn "*sub_<shift>_si_uxtw"
3128   [(set (match_operand:DI 0 "register_operand" "=r")
3129         (zero_extend:DI
3130          (minus:SI (match_operand:SI 3 "register_operand" "r")
3131                    (ASHIFT:SI
3132                     (match_operand:SI 1 "register_operand" "r")
3133                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3134   ""
3135   "sub\\t%w0, %w3, %w1, <shift> %2"
3136   [(set_attr "type" "alu_shift_imm")]
3137 )
3138
3139 (define_insn "*sub_mul_imm_<mode>"
3140   [(set (match_operand:GPI 0 "register_operand" "=r")
3141         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3142                    (mult:GPI
3143                     (match_operand:GPI 1 "register_operand" "r")
3144                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3145   ""
3146   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3147   [(set_attr "type" "alu_shift_imm")]
3148 )
3149
3150 ;; zero_extend version of above
3151 (define_insn "*sub_mul_imm_si_uxtw"
3152   [(set (match_operand:DI 0 "register_operand" "=r")
3153         (zero_extend:DI
3154          (minus:SI (match_operand:SI 3 "register_operand" "r")
3155                    (mult:SI
3156                     (match_operand:SI 1 "register_operand" "r")
3157                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3158   ""
3159   "sub\\t%w0, %w3, %w1, lsl %p2"
3160   [(set_attr "type" "alu_shift_imm")]
3161 )
3162
3163 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3164   [(set (match_operand:GPI 0 "register_operand" "=rk")
3165         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3166                    (ANY_EXTEND:GPI
3167                     (match_operand:ALLX 2 "register_operand" "r"))))]
3168   ""
3169   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3170   [(set_attr "type" "alu_ext")]
3171 )
3172
3173 ;; zero_extend version of above
3174 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3175   [(set (match_operand:DI 0 "register_operand" "=rk")
3176         (zero_extend:DI
3177          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3178                    (ANY_EXTEND:SI
3179                     (match_operand:SHORT 2 "register_operand" "r")))))]
3180   ""
3181   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3182   [(set_attr "type" "alu_ext")]
3183 )
3184
3185 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3186   [(set (match_operand:GPI 0 "register_operand" "=rk")
3187         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3188                    (ashift:GPI (ANY_EXTEND:GPI
3189                                 (match_operand:ALLX 2 "register_operand" "r"))
3190                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
3191   ""
3192   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3193   [(set_attr "type" "alu_ext")]
3194 )
3195
3196 ;; zero_extend version of above
3197 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3198   [(set (match_operand:DI 0 "register_operand" "=rk")
3199         (zero_extend:DI
3200          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3201                    (ashift:SI (ANY_EXTEND:SI
3202                                (match_operand:SHORT 2 "register_operand" "r"))
3203                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
3204   ""
3205   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3206   [(set_attr "type" "alu_ext")]
3207 )
3208
3209 (define_insn "*sub_<optab><mode>_multp2"
3210   [(set (match_operand:GPI 0 "register_operand" "=rk")
3211         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3212                    (ANY_EXTRACT:GPI
3213                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3214                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3215                     (match_operand 3 "const_int_operand" "n")
3216                     (const_int 0))))]
3217   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3218   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3219   [(set_attr "type" "alu_ext")]
3220 )
3221
3222 ;; zero_extend version of above
3223 (define_insn "*sub_<optab>si_multp2_uxtw"
3224   [(set (match_operand:DI 0 "register_operand" "=rk")
3225         (zero_extend:DI
3226          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3227                    (ANY_EXTRACT:SI
3228                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3229                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3230                     (match_operand 3 "const_int_operand" "n")
3231                     (const_int 0)))))]
3232   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3233   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3234   [(set_attr "type" "alu_ext")]
3235 )
3236
3237 ;; The hardware description is op1 + ~op2 + C.
3238 ;;                           = op1 + (-op2 + 1) + (1 - !C)
3239 ;;                           = op1 - op2 - 1 + 1 - !C
3240 ;;                           = op1 - op2 - !C.
3241 ;; We describe the latter.
3242
3243 (define_insn "*sub<mode>3_carryin0"
3244   [(set (match_operand:GPI 0 "register_operand" "=r")
3245         (minus:GPI
3246           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3247           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3248    ""
3249    "sbc\\t%<w>0, %<w>1, <w>zr"
3250   [(set_attr "type" "adc_reg")]
3251 )
3252
3253 ;; zero_extend version of the above
3254 (define_insn "*subsi3_carryin_uxtw"
3255   [(set (match_operand:DI 0 "register_operand" "=r")
3256         (zero_extend:DI
3257           (minus:SI
3258             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3259             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3260    ""
3261    "sbc\\t%w0, %w1, wzr"
3262   [(set_attr "type" "adc_reg")]
3263 )
3264
3265 (define_expand "sub<mode>3_carryin"
3266   [(set (match_operand:GPI 0 "register_operand")
3267         (minus:GPI
3268           (minus:GPI
3269             (match_operand:GPI 1 "aarch64_reg_or_zero")
3270             (match_operand:GPI 2 "register_operand"))
3271           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3272    ""
3273    ""
3274 )
3275
3276 (define_insn "*sub<mode>3_carryin"
3277   [(set (match_operand:GPI 0 "register_operand" "=r")
3278         (minus:GPI
3279           (minus:GPI
3280             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3281             (match_operand:GPI 2 "register_operand" "r"))
3282           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3283
3284    ""
3285    "sbc\\t%<w>0, %<w>1, %<w>2"
3286   [(set_attr "type" "adc_reg")]
3287 )
3288
3289 ;; zero_extend version of the above
3290 (define_insn "*subsi3_carryin_uxtw"
3291   [(set (match_operand:DI 0 "register_operand" "=r")
3292         (zero_extend:DI
3293           (minus:SI
3294             (minus:SI
3295               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3296               (match_operand:SI 2 "register_operand" "r"))
3297             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3298
3299    ""
3300    "sbc\\t%w0, %w1, %w2"
3301   [(set_attr "type" "adc_reg")]
3302 )
3303
3304 (define_insn "*sub<mode>3_carryin_alt"
3305   [(set (match_operand:GPI 0 "register_operand" "=r")
3306         (minus:GPI
3307           (minus:GPI
3308             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3309             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3310           (match_operand:GPI 2 "register_operand" "r")))]
3311    ""
3312    "sbc\\t%<w>0, %<w>1, %<w>2"
3313   [(set_attr "type" "adc_reg")]
3314 )
3315
3316 ;; zero_extend version of the above
3317 (define_insn "*subsi3_carryin_alt_uxtw"
3318   [(set (match_operand:DI 0 "register_operand" "=r")
3319         (zero_extend:DI
3320           (minus:SI
3321             (minus:SI
3322               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3323               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3324             (match_operand:SI 2 "register_operand" "r"))))]
3325    ""
3326    "sbc\\t%w0, %w1, %w2"
3327   [(set_attr "type" "adc_reg")]
3328 )
3329
3330 (define_expand "usub<GPI:mode>3_carryinC"
3331   [(parallel
3332      [(set (reg:CC CC_REGNUM)
3333            (compare:CC
3334              (zero_extend:<DWI>
3335                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3336              (plus:<DWI>
3337                (zero_extend:<DWI>
3338                  (match_operand:GPI 2 "register_operand"))
3339                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3340       (set (match_operand:GPI 0 "register_operand")
3341            (minus:GPI
3342              (minus:GPI (match_dup 1) (match_dup 2))
3343              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3344    ""
3345 )
3346
3347 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3348   [(set (reg:CC CC_REGNUM)
3349         (compare:CC
3350           (const_int 0)
3351           (plus:<DWI>
3352             (zero_extend:<DWI>
3353               (match_operand:GPI 1 "register_operand" "r"))
3354             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3355    (set (match_operand:GPI 0 "register_operand" "=r")
3356         (minus:GPI
3357           (neg:GPI (match_dup 1))
3358           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3359    ""
3360    "sbcs\\t%<w>0, <w>zr, %<w>1"
3361   [(set_attr "type" "adc_reg")]
3362 )
3363
3364 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3365   [(set (reg:CC CC_REGNUM)
3366         (compare:CC
3367           (zero_extend:<DWI>
3368             (match_operand:GPI 1 "register_operand" "r"))
3369           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3370    (set (match_operand:GPI 0 "register_operand" "=r")
3371         (minus:GPI
3372           (match_dup 1)
3373           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3374    ""
3375    "sbcs\\t%<w>0, %<w>1, <w>zr"
3376   [(set_attr "type" "adc_reg")]
3377 )
3378
3379 (define_insn "*usub<GPI:mode>3_carryinC"
3380   [(set (reg:CC CC_REGNUM)
3381         (compare:CC
3382           (zero_extend:<DWI>
3383             (match_operand:GPI 1 "register_operand" "r"))
3384           (plus:<DWI>
3385             (zero_extend:<DWI>
3386               (match_operand:GPI 2 "register_operand" "r"))
3387             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3388    (set (match_operand:GPI 0 "register_operand" "=r")
3389         (minus:GPI
3390           (minus:GPI (match_dup 1) (match_dup 2))
3391           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3392    ""
3393    "sbcs\\t%<w>0, %<w>1, %<w>2"
3394   [(set_attr "type" "adc_reg")]
3395 )
3396
3397 (define_expand "sub<GPI:mode>3_carryinV"
3398   [(parallel
3399      [(set (reg:CC_V CC_REGNUM)
3400            (compare:CC_V
3401             (minus:<DWI>
3402              (sign_extend:<DWI>
3403                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3404              (plus:<DWI>
3405                (sign_extend:<DWI>
3406                  (match_operand:GPI 2 "register_operand"))
3407                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3408             (sign_extend:<DWI>
3409              (minus:GPI (match_dup 1)
3410                         (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3411                                   (match_dup 2))))))
3412       (set (match_operand:GPI 0 "register_operand")
3413            (minus:GPI
3414              (minus:GPI (match_dup 1) (match_dup 2))
3415              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3416    ""
3417 )
3418
3419 (define_insn "*sub<mode>3_carryinV_z2"
3420   [(set (reg:CC_V CC_REGNUM)
3421         (compare:CC_V
3422          (minus:<DWI>
3423           (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3424           (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3425          (sign_extend:<DWI>
3426           (minus:GPI (match_dup 1)
3427                      (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3428    (set (match_operand:GPI 0 "register_operand" "=r")
3429         (minus:GPI
3430          (match_dup 1) (match_dup 3)))]
3431    ""
3432    "sbcs\\t%<w>0, %<w>1, <w>zr"
3433   [(set_attr "type" "adc_reg")]
3434 )
3435
3436 (define_insn "*sub<mode>3_carryinV"
3437   [(set (reg:CC_V CC_REGNUM)
3438         (compare:CC_V
3439          (minus:<DWI>
3440           (sign_extend:<DWI>
3441             (match_operand:GPI 1 "register_operand" "r"))
3442           (plus:<DWI>
3443             (sign_extend:<DWI>
3444               (match_operand:GPI 2 "register_operand" "r"))
3445             (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3446          (sign_extend:<DWI>
3447           (minus:GPI
3448            (match_dup 1)
3449            (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3450                      (match_dup 2))))))
3451    (set (match_operand:GPI 0 "register_operand" "=r")
3452         (minus:GPI
3453           (minus:GPI (match_dup 1) (match_dup 2))
3454           (match_dup 4)))]
3455    ""
3456    "sbcs\\t%<w>0, %<w>1, %<w>2"
3457   [(set_attr "type" "adc_reg")]
3458 )
3459
3460 (define_insn "*sub_uxt<mode>_shift2"
3461   [(set (match_operand:GPI 0 "register_operand" "=rk")
3462         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3463                    (and:GPI
3464                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3465                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3466                     (match_operand 3 "const_int_operand" "n"))))]
3467   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3468   "*
3469   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3470                                            INTVAL (operands[3])));
3471   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3472   [(set_attr "type" "alu_ext")]
3473 )
3474
3475 ;; zero_extend version of above
3476 (define_insn "*sub_uxtsi_shift2_uxtw"
3477   [(set (match_operand:DI 0 "register_operand" "=rk")
3478         (zero_extend:DI
3479          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3480                    (and:SI
3481                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3482                                (match_operand 2 "aarch64_imm3" "Ui3"))
3483                     (match_operand 3 "const_int_operand" "n")))))]
3484   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3485   "*
3486   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3487                                            INTVAL (operands[3])));
3488   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3489   [(set_attr "type" "alu_ext")]
3490 )
3491
3492 (define_insn "*sub_uxt<mode>_multp2"
3493   [(set (match_operand:GPI 0 "register_operand" "=rk")
3494         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3495                    (and:GPI
3496                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3497                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3498                     (match_operand 3 "const_int_operand" "n"))))]
3499   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3500   "*
3501   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3502                                            INTVAL (operands[3])));
3503   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3504   [(set_attr "type" "alu_ext")]
3505 )
3506
3507 ;; zero_extend version of above
3508 (define_insn "*sub_uxtsi_multp2_uxtw"
3509   [(set (match_operand:DI 0 "register_operand" "=rk")
3510         (zero_extend:DI
3511          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3512                    (and:SI
3513                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3514                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3515                     (match_operand 3 "const_int_operand" "n")))))]
3516   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3517   "*
3518   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3519                                            INTVAL (operands[3])));
3520   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3521   [(set_attr "type" "alu_ext")]
3522 )
3523
3524 (define_expand "abs<mode>2"
3525   [(match_operand:GPI 0 "register_operand")
3526    (match_operand:GPI 1 "register_operand")]
3527   ""
3528   {
3529     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3530     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3531     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3532     DONE;
3533   }
3534 )
3535
3536 (define_insn "neg<mode>2"
3537   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3538         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3539   ""
3540   "@
3541    neg\\t%<w>0, %<w>1
3542    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3543   [(set_attr "type" "alu_sreg, neon_neg<q>")
3544    (set_attr "arch" "*,simd")]
3545 )
3546
3547 ;; zero_extend version of above
3548 (define_insn "*negsi2_uxtw"
3549   [(set (match_operand:DI 0 "register_operand" "=r")
3550         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3551   ""
3552   "neg\\t%w0, %w1"
3553   [(set_attr "type" "alu_sreg")]
3554 )
3555
3556 (define_insn "*ngc<mode>"
3557   [(set (match_operand:GPI 0 "register_operand" "=r")
3558         (minus:GPI
3559           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3560           (match_operand:GPI 1 "register_operand" "r")))]
3561   ""
3562   "ngc\\t%<w>0, %<w>1"
3563   [(set_attr "type" "adc_reg")]
3564 )
3565
3566 (define_insn "*ngcsi_uxtw"
3567   [(set (match_operand:DI 0 "register_operand" "=r")
3568         (zero_extend:DI
3569           (minus:SI
3570             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3571             (match_operand:SI 1 "register_operand" "r"))))]
3572   ""
3573   "ngc\\t%w0, %w1"
3574   [(set_attr "type" "adc_reg")]
3575 )
3576
3577 (define_insn "neg<mode>2_compare0"
3578   [(set (reg:CC_NZ CC_REGNUM)
3579         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3580                        (const_int 0)))
3581    (set (match_operand:GPI 0 "register_operand" "=r")
3582         (neg:GPI (match_dup 1)))]
3583   ""
3584   "negs\\t%<w>0, %<w>1"
3585   [(set_attr "type" "alus_sreg")]
3586 )
3587
3588 ;; zero_extend version of above
3589 (define_insn "*negsi2_compare0_uxtw"
3590   [(set (reg:CC_NZ CC_REGNUM)
3591         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3592                        (const_int 0)))
3593    (set (match_operand:DI 0 "register_operand" "=r")
3594         (zero_extend:DI (neg:SI (match_dup 1))))]
3595   ""
3596   "negs\\t%w0, %w1"
3597   [(set_attr "type" "alus_sreg")]
3598 )
3599
3600 (define_insn "*neg_<shift><mode>3_compare0"
3601   [(set (reg:CC_NZ CC_REGNUM)
3602         (compare:CC_NZ
3603          (neg:GPI (ASHIFT:GPI
3604                    (match_operand:GPI 1 "register_operand" "r")
3605                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3606          (const_int 0)))
3607    (set (match_operand:GPI 0 "register_operand" "=r")
3608         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3609   ""
3610   "negs\\t%<w>0, %<w>1, <shift> %2"
3611   [(set_attr "type" "alus_shift_imm")]
3612 )
3613
3614 (define_insn "*neg_<shift>_<mode>2"
3615   [(set (match_operand:GPI 0 "register_operand" "=r")
3616         (neg:GPI (ASHIFT:GPI
3617                   (match_operand:GPI 1 "register_operand" "r")
3618                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3619   ""
3620   "neg\\t%<w>0, %<w>1, <shift> %2"
3621   [(set_attr "type" "alu_shift_imm")]
3622 )
3623
3624 ;; zero_extend version of above
3625 (define_insn "*neg_<shift>_si2_uxtw"
3626   [(set (match_operand:DI 0 "register_operand" "=r")
3627         (zero_extend:DI
3628          (neg:SI (ASHIFT:SI
3629                   (match_operand:SI 1 "register_operand" "r")
3630                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3631   ""
3632   "neg\\t%w0, %w1, <shift> %2"
3633   [(set_attr "type" "alu_shift_imm")]
3634 )
3635
3636 (define_insn "*neg_mul_imm_<mode>2"
3637   [(set (match_operand:GPI 0 "register_operand" "=r")
3638         (neg:GPI (mult:GPI
3639                   (match_operand:GPI 1 "register_operand" "r")
3640                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3641   ""
3642   "neg\\t%<w>0, %<w>1, lsl %p2"
3643   [(set_attr "type" "alu_shift_imm")]
3644 )
3645
3646 ;; zero_extend version of above
3647 (define_insn "*neg_mul_imm_si2_uxtw"
3648   [(set (match_operand:DI 0 "register_operand" "=r")
3649         (zero_extend:DI
3650          (neg:SI (mult:SI
3651                   (match_operand:SI 1 "register_operand" "r")
3652                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3653   ""
3654   "neg\\t%w0, %w1, lsl %p2"
3655   [(set_attr "type" "alu_shift_imm")]
3656 )
3657
3658 (define_insn "mul<mode>3"
3659   [(set (match_operand:GPI 0 "register_operand" "=r")
3660         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3661                   (match_operand:GPI 2 "register_operand" "r")))]
3662   ""
3663   "mul\\t%<w>0, %<w>1, %<w>2"
3664   [(set_attr "type" "mul")]
3665 )
3666
3667 ;; zero_extend version of above
3668 (define_insn "*mulsi3_uxtw"
3669   [(set (match_operand:DI 0 "register_operand" "=r")
3670         (zero_extend:DI
3671          (mult:SI (match_operand:SI 1 "register_operand" "r")
3672                   (match_operand:SI 2 "register_operand" "r"))))]
3673   ""
3674   "mul\\t%w0, %w1, %w2"
3675   [(set_attr "type" "mul")]
3676 )
3677
3678 (define_insn "madd<mode>"
3679   [(set (match_operand:GPI 0 "register_operand" "=r")
3680         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3681                             (match_operand:GPI 2 "register_operand" "r"))
3682                   (match_operand:GPI 3 "register_operand" "r")))]
3683   ""
3684   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3685   [(set_attr "type" "mla")]
3686 )
3687
3688 ;; zero_extend version of above
3689 (define_insn "*maddsi_uxtw"
3690   [(set (match_operand:DI 0 "register_operand" "=r")
3691         (zero_extend:DI
3692          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3693                            (match_operand:SI 2 "register_operand" "r"))
3694                   (match_operand:SI 3 "register_operand" "r"))))]
3695   ""
3696   "madd\\t%w0, %w1, %w2, %w3"
3697   [(set_attr "type" "mla")]
3698 )
3699
3700 (define_insn "*msub<mode>"
3701   [(set (match_operand:GPI 0 "register_operand" "=r")
3702         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3703                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3704                              (match_operand:GPI 2 "register_operand" "r"))))]
3705
3706   ""
3707   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3708   [(set_attr "type" "mla")]
3709 )
3710
3711 ;; zero_extend version of above
3712 (define_insn "*msubsi_uxtw"
3713   [(set (match_operand:DI 0 "register_operand" "=r")
3714         (zero_extend:DI
3715          (minus:SI (match_operand:SI 3 "register_operand" "r")
3716                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3717                             (match_operand:SI 2 "register_operand" "r")))))]
3718
3719   ""
3720   "msub\\t%w0, %w1, %w2, %w3"
3721   [(set_attr "type" "mla")]
3722 )
3723
3724 (define_insn "*mul<mode>_neg"
3725   [(set (match_operand:GPI 0 "register_operand" "=r")
3726         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3727                   (match_operand:GPI 2 "register_operand" "r")))]
3728
3729   ""
3730   "mneg\\t%<w>0, %<w>1, %<w>2"
3731   [(set_attr "type" "mul")]
3732 )
3733
3734 ;; zero_extend version of above
3735 (define_insn "*mulsi_neg_uxtw"
3736   [(set (match_operand:DI 0 "register_operand" "=r")
3737         (zero_extend:DI
3738          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3739                   (match_operand:SI 2 "register_operand" "r"))))]
3740
3741   ""
3742   "mneg\\t%w0, %w1, %w2"
3743   [(set_attr "type" "mul")]
3744 )
3745
3746 (define_insn "<su_optab>mulsidi3"
3747   [(set (match_operand:DI 0 "register_operand" "=r")
3748         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3749                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3750   ""
3751   "<su>mull\\t%0, %w1, %w2"
3752   [(set_attr "type" "<su>mull")]
3753 )
3754
3755 (define_insn "<su_optab>maddsidi4"
3756   [(set (match_operand:DI 0 "register_operand" "=r")
3757         (plus:DI (mult:DI
3758                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3759                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3760                  (match_operand:DI 3 "register_operand" "r")))]
3761   ""
3762   "<su>maddl\\t%0, %w1, %w2, %3"
3763   [(set_attr "type" "<su>mlal")]
3764 )
3765
3766 (define_insn "<su_optab>msubsidi4"
3767   [(set (match_operand:DI 0 "register_operand" "=r")
3768         (minus:DI
3769          (match_operand:DI 3 "register_operand" "r")
3770          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3771                   (ANY_EXTEND:DI
3772                    (match_operand:SI 2 "register_operand" "r")))))]
3773   ""
3774   "<su>msubl\\t%0, %w1, %w2, %3"
3775   [(set_attr "type" "<su>mlal")]
3776 )
3777
3778 (define_insn "*<su_optab>mulsidi_neg"
3779   [(set (match_operand:DI 0 "register_operand" "=r")
3780         (mult:DI (neg:DI
3781                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3782                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3783   ""
3784   "<su>mnegl\\t%0, %w1, %w2"
3785   [(set_attr "type" "<su>mull")]
3786 )
3787
3788 (define_expand "<su_optab>mulditi3"
3789   [(set (match_operand:TI 0 "register_operand")
3790         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3791                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3792   ""
3793 {
3794   rtx low = gen_reg_rtx (DImode);
3795   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3796
3797   rtx high = gen_reg_rtx (DImode);
3798   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3799
3800   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3801   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3802   DONE;
3803 })
3804
3805 ;; The default expansion of multi3 using umuldi3_highpart will perform
3806 ;; the additions in an order that fails to combine into two madd insns.
3807 (define_expand "multi3"
3808   [(set (match_operand:TI 0 "register_operand")
3809         (mult:TI (match_operand:TI 1 "register_operand")
3810                  (match_operand:TI 2 "register_operand")))]
3811   ""
3812 {
3813   rtx l0 = gen_reg_rtx (DImode);
3814   rtx l1 = gen_lowpart (DImode, operands[1]);
3815   rtx l2 = gen_lowpart (DImode, operands[2]);
3816   rtx h0 = gen_reg_rtx (DImode);
3817   rtx h1 = gen_highpart (DImode, operands[1]);
3818   rtx h2 = gen_highpart (DImode, operands[2]);
3819
3820   emit_insn (gen_muldi3 (l0, l1, l2));
3821   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3822   emit_insn (gen_madddi (h0, h1, l2, h0));
3823   emit_insn (gen_madddi (h0, l1, h2, h0));
3824
3825   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3826   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3827   DONE;
3828 })
3829
3830 (define_insn "<su>muldi3_highpart"
3831   [(set (match_operand:DI 0 "register_operand" "=r")
3832         (truncate:DI
3833          (lshiftrt:TI
3834           (mult:TI
3835            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3836            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3837           (const_int 64))))]
3838   ""
3839   "<su>mulh\\t%0, %1, %2"
3840   [(set_attr "type" "<su>mull")]
3841 )
3842
3843 (define_insn "<su_optab>div<mode>3"
3844   [(set (match_operand:GPI 0 "register_operand" "=r")
3845         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3846                      (match_operand:GPI 2 "register_operand" "r")))]
3847   ""
3848   "<su>div\\t%<w>0, %<w>1, %<w>2"
3849   [(set_attr "type" "<su>div")]
3850 )
3851
3852 ;; zero_extend version of above
3853 (define_insn "*<su_optab>divsi3_uxtw"
3854   [(set (match_operand:DI 0 "register_operand" "=r")
3855         (zero_extend:DI
3856          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3857                      (match_operand:SI 2 "register_operand" "r"))))]
3858   ""
3859   "<su>div\\t%w0, %w1, %w2"
3860   [(set_attr "type" "<su>div")]
3861 )
3862
3863 ;; -------------------------------------------------------------------
3864 ;; Comparison insns
3865 ;; -------------------------------------------------------------------
3866
3867 (define_insn "cmp<mode>"
3868   [(set (reg:CC CC_REGNUM)
3869         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3870                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3871   ""
3872   "@
3873    cmp\\t%<w>0, %<w>1
3874    cmp\\t%<w>0, %1
3875    cmn\\t%<w>0, #%n1"
3876   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3877 )
3878
3879 (define_insn "fcmp<mode>"
3880   [(set (reg:CCFP CC_REGNUM)
3881         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3882                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3883    "TARGET_FLOAT"
3884    "@
3885     fcmp\\t%<s>0, #0.0
3886     fcmp\\t%<s>0, %<s>1"
3887   [(set_attr "type" "fcmp<s>")]
3888 )
3889
3890 (define_insn "fcmpe<mode>"
3891   [(set (reg:CCFPE CC_REGNUM)
3892         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3893                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3894    "TARGET_FLOAT"
3895    "@
3896     fcmpe\\t%<s>0, #0.0
3897     fcmpe\\t%<s>0, %<s>1"
3898   [(set_attr "type" "fcmp<s>")]
3899 )
3900
3901 (define_insn "*cmp_swp_<shift>_reg<mode>"
3902   [(set (reg:CC_SWP CC_REGNUM)
3903         (compare:CC_SWP (ASHIFT:GPI
3904                          (match_operand:GPI 0 "register_operand" "r")
3905                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3906                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3907   ""
3908   "cmp\\t%<w>2, %<w>0, <shift> %1"
3909   [(set_attr "type" "alus_shift_imm")]
3910 )
3911
3912 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3913   [(set (reg:CC_SWP CC_REGNUM)
3914         (compare:CC_SWP (ANY_EXTEND:GPI
3915                          (match_operand:ALLX 0 "register_operand" "r"))
3916                         (match_operand:GPI 1 "register_operand" "r")))]
3917   ""
3918   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3919   [(set_attr "type" "alus_ext")]
3920 )
3921
3922 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3923   [(set (reg:CC_SWP CC_REGNUM)
3924         (compare:CC_SWP (ashift:GPI
3925                          (ANY_EXTEND:GPI
3926                           (match_operand:ALLX 0 "register_operand" "r"))
3927                          (match_operand 1 "aarch64_imm3" "Ui3"))
3928         (match_operand:GPI 2 "register_operand" "r")))]
3929   ""
3930   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3931   [(set_attr "type" "alus_ext")]
3932 )
3933
3934 ;; -------------------------------------------------------------------
3935 ;; Store-flag and conditional select insns
3936 ;; -------------------------------------------------------------------
3937
3938 (define_expand "cstore<mode>4"
3939   [(set (match_operand:SI 0 "register_operand")
3940         (match_operator:SI 1 "aarch64_comparison_operator"
3941          [(match_operand:GPI 2 "register_operand")
3942           (match_operand:GPI 3 "aarch64_plus_operand")]))]
3943   ""
3944   "
3945   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3946                                       operands[3]);
3947   operands[3] = const0_rtx;
3948   "
3949 )
3950
3951 (define_expand "cstorecc4"
3952   [(set (match_operand:SI 0 "register_operand")
3953        (match_operator 1 "aarch64_comparison_operator_mode"
3954         [(match_operand 2 "cc_register")
3955          (match_operand 3 "const0_operand")]))]
3956   ""
3957 "{
3958   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3959   DONE;
3960 }")
3961
3962
3963 (define_expand "cstore<mode>4"
3964   [(set (match_operand:SI 0 "register_operand")
3965         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3966          [(match_operand:GPF 2 "register_operand")
3967           (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
3968   ""
3969   "
3970   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3971                                       operands[3]);
3972   operands[3] = const0_rtx;
3973   "
3974 )
3975
3976 (define_insn "aarch64_cstore<mode>"
3977   [(set (match_operand:ALLI 0 "register_operand" "=r")
3978         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3979          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3980   ""
3981   "cset\\t%<w>0, %m1"
3982   [(set_attr "type" "csel")]
3983 )
3984
3985 ;; For a 24-bit immediate CST we can optimize the compare for equality
3986 ;; and branch sequence from:
3987 ;;      mov     x0, #imm1
3988 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3989 ;;      cmp     x1, x0
3990 ;;      cset    x2, <ne,eq>
3991 ;; into the shorter:
3992 ;;      sub     x0, x1, #(CST & 0xfff000)
3993 ;;      subs    x0, x0, #(CST & 0x000fff)
3994 ;;      cset x2, <ne, eq>.
3995 (define_insn_and_split "*compare_cstore<mode>_insn"
3996   [(set (match_operand:GPI 0 "register_operand" "=r")
3997          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3998                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3999    (clobber (reg:CC CC_REGNUM))]
4000   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4001    && !aarch64_plus_operand (operands[2], <MODE>mode)
4002    && !reload_completed"
4003   "#"
4004   "&& true"
4005   [(const_int 0)]
4006   {
4007     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4008     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4009     rtx tmp = gen_reg_rtx (<MODE>mode);
4010     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4011     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4012     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4013     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4014     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4015     DONE;
4016   }
4017   [(set_attr "type" "csel")]
4018 )
4019
4020 ;; zero_extend version of the above
4021 (define_insn "*cstoresi_insn_uxtw"
4022   [(set (match_operand:DI 0 "register_operand" "=r")
4023         (zero_extend:DI
4024          (match_operator:SI 1 "aarch64_comparison_operator_mode"
4025           [(match_operand 2 "cc_register" "") (const_int 0)])))]
4026   ""
4027   "cset\\t%w0, %m1"
4028   [(set_attr "type" "csel")]
4029 )
4030
4031 (define_insn "cstore<mode>_neg"
4032   [(set (match_operand:ALLI 0 "register_operand" "=r")
4033         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4034                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
4035   ""
4036   "csetm\\t%<w>0, %m1"
4037   [(set_attr "type" "csel")]
4038 )
4039
4040 ;; zero_extend version of the above
4041 (define_insn "*cstoresi_neg_uxtw"
4042   [(set (match_operand:DI 0 "register_operand" "=r")
4043         (zero_extend:DI
4044          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4045                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4046   ""
4047   "csetm\\t%w0, %m1"
4048   [(set_attr "type" "csel")]
4049 )
4050
4051 (define_expand "cmov<mode>6"
4052   [(set (match_operand:GPI 0 "register_operand")
4053         (if_then_else:GPI
4054          (match_operator 1 "aarch64_comparison_operator"
4055           [(match_operand:GPI 2 "register_operand")
4056            (match_operand:GPI 3 "aarch64_plus_operand")])
4057          (match_operand:GPI 4 "register_operand")
4058          (match_operand:GPI 5 "register_operand")))]
4059   ""
4060   "
4061   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4062                                       operands[3]);
4063   operands[3] = const0_rtx;
4064   "
4065 )
4066
4067 (define_expand "cmov<mode>6"
4068   [(set (match_operand:GPF 0 "register_operand")
4069         (if_then_else:GPF
4070          (match_operator 1 "aarch64_comparison_operator"
4071           [(match_operand:GPF 2 "register_operand")
4072            (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4073          (match_operand:GPF 4 "register_operand")
4074          (match_operand:GPF 5 "register_operand")))]
4075   ""
4076   "
4077   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4078                                       operands[3]);
4079   operands[3] = const0_rtx;
4080   "
4081 )
4082
4083 (define_insn "*cmov<mode>_insn"
4084   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4085         (if_then_else:ALLI
4086          (match_operator 1 "aarch64_comparison_operator"
4087           [(match_operand 2 "cc_register" "") (const_int 0)])
4088          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4089          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4090   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4091      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4092   ;; Final two alternatives should be unreachable, but included for completeness
4093   "@
4094    csel\\t%<w>0, %<w>3, %<w>4, %m1
4095    csinv\\t%<w>0, %<w>3, <w>zr, %m1
4096    csinv\\t%<w>0, %<w>4, <w>zr, %M1
4097    csinc\\t%<w>0, %<w>3, <w>zr, %m1
4098    csinc\\t%<w>0, %<w>4, <w>zr, %M1
4099    mov\\t%<w>0, -1
4100    mov\\t%<w>0, 1"
4101   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4102 )
4103
4104 ;; zero_extend version of above
4105 (define_insn "*cmovsi_insn_uxtw"
4106   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4107         (zero_extend:DI
4108          (if_then_else:SI
4109           (match_operator 1 "aarch64_comparison_operator"
4110            [(match_operand 2 "cc_register" "") (const_int 0)])
4111           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4112           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4113   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4114      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4115   ;; Final two alternatives should be unreachable, but included for completeness
4116   "@
4117    csel\\t%w0, %w3, %w4, %m1
4118    csinv\\t%w0, %w3, wzr, %m1
4119    csinv\\t%w0, %w4, wzr, %M1
4120    csinc\\t%w0, %w3, wzr, %m1
4121    csinc\\t%w0, %w4, wzr, %M1
4122    mov\\t%w0, -1
4123    mov\\t%w0, 1"
4124   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4125 )
4126
4127 (define_insn "*cmovdi_insn_uxtw"
4128   [(set (match_operand:DI 0 "register_operand" "=r")
4129         (if_then_else:DI
4130          (match_operator 1 "aarch64_comparison_operator"
4131           [(match_operand 2 "cc_register" "") (const_int 0)])
4132          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4133          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4134   ""
4135   "csel\\t%w0, %w3, %w4, %m1"
4136   [(set_attr "type" "csel")]
4137 )
4138
4139 (define_insn "*cmov<mode>_insn"
4140   [(set (match_operand:GPF 0 "register_operand" "=w")
4141         (if_then_else:GPF
4142          (match_operator 1 "aarch64_comparison_operator"
4143           [(match_operand 2 "cc_register" "") (const_int 0)])
4144          (match_operand:GPF 3 "register_operand" "w")
4145          (match_operand:GPF 4 "register_operand" "w")))]
4146   "TARGET_FLOAT"
4147   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4148   [(set_attr "type" "fcsel")]
4149 )
4150
4151 (define_expand "mov<mode>cc"
4152   [(set (match_operand:ALLI 0 "register_operand")
4153         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4154                            (match_operand:ALLI 2 "register_operand")
4155                            (match_operand:ALLI 3 "register_operand")))]
4156   ""
4157   {
4158     rtx ccreg;
4159     enum rtx_code code = GET_CODE (operands[1]);
4160
4161     if (code == UNEQ || code == LTGT)
4162       FAIL;
4163
4164     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4165                                      XEXP (operands[1], 1));
4166     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4167   }
4168 )
4169
4170 (define_expand "mov<GPF:mode><GPI:mode>cc"
4171   [(set (match_operand:GPI 0 "register_operand")
4172         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4173                           (match_operand:GPF 2 "register_operand")
4174                           (match_operand:GPF 3 "register_operand")))]
4175   ""
4176   {
4177     rtx ccreg;
4178     enum rtx_code code = GET_CODE (operands[1]);
4179
4180     if (code == UNEQ || code == LTGT)
4181       FAIL;
4182
4183     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4184                                   XEXP (operands[1], 1));
4185     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4186   }
4187 )
4188
4189 (define_expand "mov<mode>cc"
4190   [(set (match_operand:GPF 0 "register_operand")
4191         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4192                           (match_operand:GPF 2 "register_operand")
4193                           (match_operand:GPF 3 "register_operand")))]
4194   ""
4195   {
4196     rtx ccreg;
4197     enum rtx_code code = GET_CODE (operands[1]);
4198
4199     if (code == UNEQ || code == LTGT)
4200       FAIL;
4201
4202     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4203                                   XEXP (operands[1], 1));
4204     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4205   }
4206 )
4207
4208 (define_expand "<neg_not_op><mode>cc"
4209   [(set (match_operand:GPI 0 "register_operand")
4210         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4211                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4212                           (match_operand:GPI 3 "register_operand")))]
4213   ""
4214   {
4215     rtx ccreg;
4216     enum rtx_code code = GET_CODE (operands[1]);
4217
4218     if (code == UNEQ || code == LTGT)
4219       FAIL;
4220
4221     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4222                                       XEXP (operands[1], 1));
4223     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4224   }
4225 )
4226
4227 ;; CRC32 instructions.
4228 (define_insn "aarch64_<crc_variant>"
4229   [(set (match_operand:SI 0 "register_operand" "=r")
4230         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4231                     (match_operand:<crc_mode> 2 "register_operand" "r")]
4232          CRC))]
4233   "TARGET_CRC32"
4234   {
4235     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4236       return "<crc_variant>\\t%w0, %w1, %x2";
4237     else
4238       return "<crc_variant>\\t%w0, %w1, %w2";
4239   }
4240   [(set_attr "type" "crc")]
4241 )
4242
4243 (define_insn "*csinc2<mode>_insn"
4244   [(set (match_operand:GPI 0 "register_operand" "=r")
4245         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4246                   (match_operand:GPI 1 "register_operand" "r")))]
4247   ""
4248   "cinc\\t%<w>0, %<w>1, %m2"
4249   [(set_attr "type" "csel")]
4250 )
4251
4252 (define_insn "csinc3<mode>_insn"
4253   [(set (match_operand:GPI 0 "register_operand" "=r")
4254         (if_then_else:GPI
4255           (match_operand 1 "aarch64_comparison_operation" "")
4256           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4257                     (const_int 1))
4258           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4259   ""
4260   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4261   [(set_attr "type" "csel")]
4262 )
4263
4264 (define_insn "*csinv3<mode>_insn"
4265   [(set (match_operand:GPI 0 "register_operand" "=r")
4266         (if_then_else:GPI
4267           (match_operand 1 "aarch64_comparison_operation" "")
4268           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4269           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4270   ""
4271   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4272   [(set_attr "type" "csel")]
4273 )
4274
4275 (define_insn "csneg3_uxtw_insn"
4276   [(set (match_operand:DI 0 "register_operand" "=r")
4277         (zero_extend:DI
4278           (if_then_else:SI
4279             (match_operand 1 "aarch64_comparison_operation" "")
4280             (neg:SI (match_operand:SI 2 "register_operand" "r"))
4281             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4282   ""
4283   "csneg\\t%w0, %w3, %w2, %M1"
4284   [(set_attr "type" "csel")]
4285 )
4286
4287 (define_insn "csneg3<mode>_insn"
4288   [(set (match_operand:GPI 0 "register_operand" "=r")
4289         (if_then_else:GPI
4290           (match_operand 1 "aarch64_comparison_operation" "")
4291           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4292           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4293   ""
4294   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4295   [(set_attr "type" "csel")]
4296 )
4297
4298 ;; If X can be loaded by a single CNT[BHWD] instruction,
4299 ;;
4300 ;;    A = UMAX (B, X)
4301 ;;
4302 ;; is equivalent to:
4303 ;;
4304 ;;    TMP = UQDEC[BHWD] (B, X)
4305 ;;    A = TMP + X
4306 ;;
4307 ;; Defining the pattern this way means that:
4308 ;;
4309 ;;    A = UMAX (B, X) - X
4310 ;;
4311 ;; becomes:
4312 ;;
4313 ;;    TMP1 = UQDEC[BHWD] (B, X)
4314 ;;    TMP2 = TMP1 + X
4315 ;;    A = TMP2 - X
4316 ;;
4317 ;; which combine can optimize to:
4318 ;;
4319 ;;    A = UQDEC[BHWD] (B, X)
4320 ;;
4321 ;; We don't use match_operand predicates because the order of the operands
4322 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4323 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4324 (define_expand "umax<mode>3"
4325   [(set (match_operand:GPI 0 "register_operand")
4326         (umax:GPI (match_operand:GPI 1 "")
4327                   (match_operand:GPI 2 "")))]
4328   "TARGET_SVE"
4329   {
4330     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4331       std::swap (operands[1], operands[2]);
4332     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4333       FAIL;
4334     rtx temp = gen_reg_rtx (<MODE>mode);
4335     operands[1] = force_reg (<MODE>mode, operands[1]);
4336     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4337     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4338     DONE;
4339   }
4340 )
4341
4342 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4343 (define_insn "aarch64_uqdec<mode>"
4344   [(set (match_operand:GPI 0 "register_operand" "=r")
4345         (minus:GPI
4346          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4347                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4348          (match_dup 2)))]
4349   "TARGET_SVE"
4350   {
4351     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4352   }
4353 )
4354
4355 ;; -------------------------------------------------------------------
4356 ;; Logical operations
4357 ;; -------------------------------------------------------------------
4358
4359
4360 (define_insn_and_split "*aarch64_and<mode>_imm2"
4361   [(set (match_operand:GPI 0 "register_operand" "=rk")
4362         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4363                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4364   ""
4365   "#"
4366   "true"
4367   [(const_int 0)]
4368   {
4369      HOST_WIDE_INT val = INTVAL (operands[2]);
4370      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4371      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4372
4373      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4374      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4375      DONE;
4376   }
4377 )
4378
4379 (define_insn "<optab><mode>3"
4380   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4381         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4382                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4383   ""
4384   "@
4385   <logical>\\t%<w>0, %<w>1, %<w>2
4386   <logical>\\t%<w>0, %<w>1, %2
4387   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4388   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4389    (set_attr "arch" "*,*,simd")]
4390 )
4391
4392 ;; zero_extend version of above
4393 (define_insn "*<optab>si3_uxtw"
4394   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4395         (zero_extend:DI
4396          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4397                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4398   ""
4399   "@
4400    <logical>\\t%w0, %w1, %w2
4401    <logical>\\t%w0, %w1, %2"
4402   [(set_attr "type" "logic_reg,logic_imm")]
4403 )
4404
4405 (define_insn "*and<mode>3_compare0"
4406   [(set (reg:CC_NZ CC_REGNUM)
4407         (compare:CC_NZ
4408          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4409                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4410          (const_int 0)))
4411    (set (match_operand:GPI 0 "register_operand" "=r,r")
4412         (and:GPI (match_dup 1) (match_dup 2)))]
4413   ""
4414   "@
4415    ands\\t%<w>0, %<w>1, %<w>2
4416    ands\\t%<w>0, %<w>1, %2"
4417   [(set_attr "type" "logics_reg,logics_imm")]
4418 )
4419
4420 ;; zero_extend version of above
4421 (define_insn "*andsi3_compare0_uxtw"
4422   [(set (reg:CC_NZ CC_REGNUM)
4423         (compare:CC_NZ
4424          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4425                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4426          (const_int 0)))
4427    (set (match_operand:DI 0 "register_operand" "=r,r")
4428         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4429   ""
4430   "@
4431    ands\\t%w0, %w1, %w2
4432    ands\\t%w0, %w1, %2"
4433   [(set_attr "type" "logics_reg,logics_imm")]
4434 )
4435
4436 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4437   [(set (reg:CC_NZ CC_REGNUM)
4438         (compare:CC_NZ
4439          (and:GPI (SHIFT:GPI
4440                    (match_operand:GPI 1 "register_operand" "r")
4441                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4442                   (match_operand:GPI 3 "register_operand" "r"))
4443          (const_int 0)))
4444    (set (match_operand:GPI 0 "register_operand" "=r")
4445         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4446   ""
4447   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4448   [(set_attr "type" "logics_shift_imm")]
4449 )
4450
4451 ;; zero_extend version of above
4452 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4453   [(set (reg:CC_NZ CC_REGNUM)
4454         (compare:CC_NZ
4455          (and:SI (SHIFT:SI
4456                   (match_operand:SI 1 "register_operand" "r")
4457                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4458                  (match_operand:SI 3 "register_operand" "r"))
4459          (const_int 0)))
4460    (set (match_operand:DI 0 "register_operand" "=r")
4461         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4462                                 (match_dup 3))))]
4463   ""
4464   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4465   [(set_attr "type" "logics_shift_imm")]
4466 )
4467
4468 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4469   [(set (match_operand:GPI 0 "register_operand" "=r")
4470         (LOGICAL:GPI (SHIFT:GPI
4471                       (match_operand:GPI 1 "register_operand" "r")
4472                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4473                      (match_operand:GPI 3 "register_operand" "r")))]
4474   ""
4475   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4476   [(set_attr "type" "logic_shift_imm")]
4477 )
4478
4479 (define_insn "*<optab>_rol<mode>3"
4480   [(set (match_operand:GPI 0 "register_operand" "=r")
4481         (LOGICAL:GPI (rotate:GPI
4482                       (match_operand:GPI 1 "register_operand" "r")
4483                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4484                      (match_operand:GPI 3 "register_operand" "r")))]
4485   ""
4486   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4487   [(set_attr "type" "logic_shift_imm")]
4488 )
4489
4490 ;; zero_extend versions of above
4491 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4492   [(set (match_operand:DI 0 "register_operand" "=r")
4493         (zero_extend:DI
4494          (LOGICAL:SI (SHIFT:SI
4495                       (match_operand:SI 1 "register_operand" "r")
4496                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4497                      (match_operand:SI 3 "register_operand" "r"))))]
4498   ""
4499   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4500   [(set_attr "type" "logic_shift_imm")]
4501 )
4502
4503 (define_insn "*<optab>_rolsi3_uxtw"
4504   [(set (match_operand:DI 0 "register_operand" "=r")
4505         (zero_extend:DI
4506          (LOGICAL:SI (rotate:SI
4507                       (match_operand:SI 1 "register_operand" "r")
4508                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4509                      (match_operand:SI 3 "register_operand" "r"))))]
4510   ""
4511   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4512   [(set_attr "type" "logic_shift_imm")]
4513 )
4514
4515 (define_insn "one_cmpl<mode>2"
4516   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4517         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4518   ""
4519   "@
4520   mvn\\t%<w>0, %<w>1
4521   mvn\\t%0.8b, %1.8b"
4522   [(set_attr "type" "logic_reg,neon_logic")
4523    (set_attr "arch" "*,simd")]
4524 )
4525
4526 (define_insn "*one_cmpl_<optab><mode>2"
4527   [(set (match_operand:GPI 0 "register_operand" "=r")
4528         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4529                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4530   ""
4531   "mvn\\t%<w>0, %<w>1, <shift> %2"
4532   [(set_attr "type" "logic_shift_imm")]
4533 )
4534
4535 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4536
4537 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4538   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4539         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4540                      (match_operand:GPI 2 "register_operand" "r,w")))]
4541   ""
4542   "@
4543   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4544   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4545   [(set_attr "type" "logic_reg,neon_logic")
4546    (set_attr "arch" "*,simd")]
4547 )
4548
4549 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4550   [(set (match_operand:DI 0 "register_operand" "=r")
4551         (zero_extend:DI
4552           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4553                        (match_operand:SI 2 "register_operand" "r"))))]
4554   ""
4555   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4556   [(set_attr "type" "logic_reg")]
4557 )
4558
4559 (define_insn "*xor_one_cmplsidi3_ze"
4560   [(set (match_operand:DI 0 "register_operand" "=r")
4561         (zero_extend:DI
4562           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4563                           (match_operand:SI 2 "register_operand" "r")))))]
4564   ""
4565   "eon\\t%w0, %w1, %w2"
4566   [(set_attr "type" "logic_reg")]
4567 )
4568
4569 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4570 ;; eon does not operate on SIMD registers so the vector variant must be split.
4571 (define_insn_and_split "*xor_one_cmpl<mode>3"
4572   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4573         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4574                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4575   ""
4576   "@
4577   eon\\t%<w>0, %<w>1, %<w>2
4578   #"
4579   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4580   [(set (match_operand:GPI 0 "register_operand" "=w")
4581         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4582                  (match_operand:GPI 2 "register_operand" "w")))
4583    (set (match_dup 0) (not:GPI (match_dup 0)))]
4584   ""
4585   [(set_attr "type" "logic_reg,multiple")
4586    (set_attr "arch" "*,simd")]
4587 )
4588
4589 (define_insn "*and_one_cmpl<mode>3_compare0"
4590   [(set (reg:CC_NZ CC_REGNUM)
4591         (compare:CC_NZ
4592          (and:GPI (not:GPI
4593                    (match_operand:GPI 1 "register_operand" "r"))
4594                   (match_operand:GPI 2 "register_operand" "r"))
4595          (const_int 0)))
4596    (set (match_operand:GPI 0 "register_operand" "=r")
4597         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4598   ""
4599   "bics\\t%<w>0, %<w>2, %<w>1"
4600   [(set_attr "type" "logics_reg")]
4601 )
4602
4603 ;; zero_extend version of above
4604 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4605   [(set (reg:CC_NZ CC_REGNUM)
4606         (compare:CC_NZ
4607          (and:SI (not:SI
4608                   (match_operand:SI 1 "register_operand" "r"))
4609                  (match_operand:SI 2 "register_operand" "r"))
4610          (const_int 0)))
4611    (set (match_operand:DI 0 "register_operand" "=r")
4612         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4613   ""
4614   "bics\\t%w0, %w2, %w1"
4615   [(set_attr "type" "logics_reg")]
4616 )
4617
4618 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4619   [(set (reg:CC_NZ CC_REGNUM)
4620     (compare:CC_NZ
4621      (and:GPI (not:GPI
4622            (match_operand:GPI 0 "register_operand" "r"))
4623           (match_operand:GPI 1 "register_operand" "r"))
4624      (const_int 0)))]
4625   ""
4626   "bics\\t<w>zr, %<w>1, %<w>0"
4627   [(set_attr "type" "logics_reg")]
4628 )
4629
4630 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4631   [(set (match_operand:GPI 0 "register_operand" "=r")
4632         (LOGICAL:GPI (not:GPI
4633                       (SHIFT:GPI
4634                        (match_operand:GPI 1 "register_operand" "r")
4635                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4636                      (match_operand:GPI 3 "register_operand" "r")))]
4637   ""
4638   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4639   [(set_attr "type" "logic_shift_imm")]
4640 )
4641
4642 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4643   [(set (match_operand:GPI 0 "register_operand" "=r")
4644         (not:GPI (xor:GPI
4645                       (SHIFT:GPI
4646                        (match_operand:GPI 1 "register_operand" "r")
4647                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4648                      (match_operand:GPI 3 "register_operand" "r"))))]
4649   ""
4650   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4651   [(set_attr "type" "logic_shift_imm")]
4652 )
4653
4654 ;; Zero-extend version of the above.
4655 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4656   [(set (match_operand:DI 0 "register_operand" "=r")
4657         (zero_extend:DI
4658           (not:SI (xor:SI
4659                     (SHIFT:SI
4660                       (match_operand:SI 1 "register_operand" "r")
4661                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4662                     (match_operand:SI 3 "register_operand" "r")))))]
4663   ""
4664   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4665   [(set_attr "type" "logic_shift_imm")]
4666 )
4667
4668 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4669   [(set (reg:CC_NZ CC_REGNUM)
4670         (compare:CC_NZ
4671          (and:GPI (not:GPI
4672                    (SHIFT:GPI
4673                     (match_operand:GPI 1 "register_operand" "r")
4674                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4675                   (match_operand:GPI 3 "register_operand" "r"))
4676          (const_int 0)))
4677    (set (match_operand:GPI 0 "register_operand" "=r")
4678         (and:GPI (not:GPI
4679                   (SHIFT:GPI
4680                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4681   ""
4682   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4683   [(set_attr "type" "logics_shift_imm")]
4684 )
4685
4686 ;; zero_extend version of above
4687 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4688   [(set (reg:CC_NZ CC_REGNUM)
4689         (compare:CC_NZ
4690          (and:SI (not:SI
4691                   (SHIFT:SI
4692                    (match_operand:SI 1 "register_operand" "r")
4693                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4694                  (match_operand:SI 3 "register_operand" "r"))
4695          (const_int 0)))
4696    (set (match_operand:DI 0 "register_operand" "=r")
4697         (zero_extend:DI (and:SI
4698                          (not:SI
4699                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4700   ""
4701   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4702   [(set_attr "type" "logics_shift_imm")]
4703 )
4704
4705 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4706   [(set (reg:CC_NZ CC_REGNUM)
4707     (compare:CC_NZ
4708      (and:GPI (not:GPI
4709            (SHIFT:GPI
4710             (match_operand:GPI 0 "register_operand" "r")
4711             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4712           (match_operand:GPI 2 "register_operand" "r"))
4713      (const_int 0)))]
4714   ""
4715   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4716   [(set_attr "type" "logics_shift_imm")]
4717 )
4718
4719 (define_insn "clz<mode>2"
4720   [(set (match_operand:GPI 0 "register_operand" "=r")
4721         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4722   ""
4723   "clz\\t%<w>0, %<w>1"
4724   [(set_attr "type" "clz")]
4725 )
4726
4727 (define_expand "ffs<mode>2"
4728   [(match_operand:GPI 0 "register_operand")
4729    (match_operand:GPI 1 "register_operand")]
4730   ""
4731   {
4732     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4733     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4734
4735     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4736     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4737     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4738     DONE;
4739   }
4740 )
4741
4742 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4743 ;;
4744 ;; MOV  v.1d, x0
4745 ;; CNT  v1.8b, v.8b
4746 ;; ADDV b2, v1.8b
4747 ;; MOV  w0, v2.b[0]
4748
4749 (define_expand "popcount<mode>2"
4750   [(match_operand:GPI 0 "register_operand")
4751    (match_operand:GPI 1 "register_operand")]
4752   "TARGET_SIMD"
4753 {
4754   rtx v = gen_reg_rtx (V8QImode);
4755   rtx v1 = gen_reg_rtx (V8QImode);
4756   rtx r = gen_reg_rtx (QImode);
4757   rtx in = operands[1];
4758   rtx out = operands[0];
4759   if(<MODE>mode == SImode)
4760     {
4761       rtx tmp;
4762       tmp = gen_reg_rtx (DImode);
4763       /* If we have SImode, zero extend to DImode, pop count does
4764          not change if we have extra zeros. */
4765       emit_insn (gen_zero_extendsidi2 (tmp, in));
4766       in = tmp;
4767     }
4768   emit_move_insn (v, gen_lowpart (V8QImode, in));
4769   emit_insn (gen_popcountv8qi2 (v1, v));
4770   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4771   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4772   DONE;
4773 })
4774
4775 (define_insn "clrsb<mode>2"
4776   [(set (match_operand:GPI 0 "register_operand" "=r")
4777         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4778   ""
4779   "cls\\t%<w>0, %<w>1"
4780   [(set_attr "type" "clz")]
4781 )
4782
4783 (define_insn "rbit<mode>2"
4784   [(set (match_operand:GPI 0 "register_operand" "=r")
4785         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4786   ""
4787   "rbit\\t%<w>0, %<w>1"
4788   [(set_attr "type" "rbit")]
4789 )
4790
4791 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4792 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4793 ;; expression and split after reload to enable scheduling them apart if
4794 ;; needed.
4795
4796 (define_insn_and_split "ctz<mode>2"
4797  [(set (match_operand:GPI           0 "register_operand" "=r")
4798        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4799   ""
4800   "#"
4801   "reload_completed"
4802   [(const_int 0)]
4803   "
4804   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4805   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4806   DONE;
4807 ")
4808
4809 (define_insn "*and<mode>_compare0"
4810   [(set (reg:CC_NZ CC_REGNUM)
4811         (compare:CC_NZ
4812          (match_operand:SHORT 0 "register_operand" "r")
4813          (const_int 0)))]
4814   ""
4815   "tst\\t%<w>0, <short_mask>"
4816   [(set_attr "type" "alus_imm")]
4817 )
4818
4819 (define_insn "*ands<GPI:mode>_compare0"
4820   [(set (reg:CC_NZ CC_REGNUM)
4821         (compare:CC_NZ
4822          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4823          (const_int 0)))
4824    (set (match_operand:GPI 0 "register_operand" "=r")
4825         (zero_extend:GPI (match_dup 1)))]
4826   ""
4827   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4828   [(set_attr "type" "alus_imm")]
4829 )
4830
4831 (define_insn "*and<mode>3nr_compare0"
4832   [(set (reg:CC_NZ CC_REGNUM)
4833         (compare:CC_NZ
4834          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4835                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4836          (const_int 0)))]
4837   ""
4838   "@
4839    tst\\t%<w>0, %<w>1
4840    tst\\t%<w>0, %1"
4841   [(set_attr "type" "logics_reg,logics_imm")]
4842 )
4843
4844 (define_split
4845   [(set (reg:CC_NZ CC_REGNUM)
4846         (compare:CC_NZ
4847          (and:GPI (match_operand:GPI 0 "register_operand")
4848                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4849          (const_int 0)))
4850    (clobber (match_operand:SI 2 "register_operand"))]
4851   ""
4852   [(set (match_dup 2) (match_dup 1))
4853    (set (reg:CC_NZ CC_REGNUM)
4854         (compare:CC_NZ
4855          (and:GPI (match_dup 0)
4856                   (match_dup 2))
4857          (const_int 0)))]
4858 )
4859
4860 (define_insn "*and<mode>3nr_compare0_zextract"
4861   [(set (reg:CC_NZ CC_REGNUM)
4862         (compare:CC_NZ
4863          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4864                   (match_operand:GPI 1 "const_int_operand" "n")
4865                   (match_operand:GPI 2 "const_int_operand" "n"))
4866          (const_int 0)))]
4867   "INTVAL (operands[1]) > 0
4868    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4869         <= GET_MODE_BITSIZE (<MODE>mode))
4870    && aarch64_bitmask_imm (
4871         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4872                                                  operands[2])),
4873         <MODE>mode)"
4874   {
4875     operands[1]
4876       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4877     return "tst\\t%<w>0, %1";
4878   }
4879   [(set_attr "type" "logics_shift_imm")]
4880 )
4881
4882 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4883   [(set (reg:CC_NZ CC_REGNUM)
4884         (compare:CC_NZ
4885          (and:GPI (SHIFT:GPI
4886                    (match_operand:GPI 0 "register_operand" "r")
4887                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4888                   (match_operand:GPI 2 "register_operand" "r"))
4889         (const_int 0)))]
4890   ""
4891   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4892   [(set_attr "type" "logics_shift_imm")]
4893 )
4894
4895 (define_split
4896   [(set (reg:CC_NZ CC_REGNUM)
4897         (compare:CC_NZ
4898          (and:GPI (SHIFT:GPI
4899                    (match_operand:GPI 0 "register_operand")
4900                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4901                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4902         (const_int 0)))
4903     (clobber (match_operand:SI 3 "register_operand"))]
4904   ""
4905   [(set (match_dup 3) (match_dup 2))
4906    (set (reg:CC_NZ CC_REGNUM)
4907         (compare:CC_NZ
4908          (and:GPI (SHIFT:GPI
4909                    (match_dup 0)
4910                    (match_dup 1))
4911                   (match_dup 3))
4912          (const_int 0)))]
4913 )
4914
4915 ;; -------------------------------------------------------------------
4916 ;; Shifts
4917 ;; -------------------------------------------------------------------
4918
4919 (define_expand "<optab><mode>3"
4920   [(set (match_operand:GPI 0 "register_operand")
4921         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4922                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4923   ""
4924   {
4925     if (CONST_INT_P (operands[2]))
4926       {
4927         operands[2] = GEN_INT (INTVAL (operands[2])
4928                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4929
4930         if (operands[2] == const0_rtx)
4931           {
4932             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4933             DONE;
4934           }
4935       }
4936   }
4937 )
4938
4939 (define_expand "ashl<mode>3"
4940   [(set (match_operand:SHORT 0 "register_operand")
4941         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4942                       (match_operand:QI 2 "const_int_operand")))]
4943   ""
4944   {
4945     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4946
4947     if (operands[2] == const0_rtx)
4948       {
4949         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4950         DONE;
4951       }
4952   }
4953 )
4954
4955 (define_expand "rotr<mode>3"
4956   [(set (match_operand:GPI 0 "register_operand")
4957         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4958                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4959   ""
4960   {
4961     if (CONST_INT_P (operands[2]))
4962       {
4963         operands[2] = GEN_INT (INTVAL (operands[2])
4964                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4965
4966         if (operands[2] == const0_rtx)
4967           {
4968             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4969             DONE;
4970           }
4971       }
4972   }
4973 )
4974
4975 (define_expand "rotl<mode>3"
4976   [(set (match_operand:GPI 0 "register_operand")
4977         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4978                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4979   ""
4980   {
4981     /* (SZ - cnt) % SZ == -cnt % SZ */
4982     if (CONST_INT_P (operands[2]))
4983       {
4984         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4985                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4986         if (operands[2] == const0_rtx)
4987           {
4988             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4989             DONE;
4990           }
4991       }
4992     else
4993       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4994                                         NULL_RTX, 1);
4995   }
4996 )
4997
4998 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4999 ;; they truncate the shift/rotate amount by the size of the registers they
5000 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
5001 ;; such redundant masking instructions.  GCC can do that automatically when
5002 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5003 ;; because some of the SISD shift alternatives don't perform this truncations.
5004 ;; So this pattern exists to catch such cases.
5005
5006 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5007   [(set (match_operand:GPI 0 "register_operand" "=r")
5008         (SHIFT:GPI
5009           (match_operand:GPI 1 "register_operand" "r")
5010           (match_operator 4 "subreg_lowpart_operator"
5011            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5012                      (match_operand 3 "const_int_operand" "n"))])))]
5013   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5014   "<shift>\t%<w>0, %<w>1, %<w>2"
5015   [(set_attr "type" "shift_reg")]
5016 )
5017
5018 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5019   [(set (match_operand:GPI 0 "register_operand" "=&r")
5020         (SHIFT:GPI
5021           (match_operand:GPI 1 "register_operand" "r")
5022           (match_operator 4 "subreg_lowpart_operator"
5023           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5024                            (match_operand 3 "const_int_operand" "n")))])))]
5025   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5026   "#"
5027   "&& true"
5028   [(const_int 0)]
5029   {
5030     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5031                : lowpart_subreg (SImode, operands[0], <MODE>mode));
5032     emit_insn (gen_negsi2 (tmp, operands[2]));
5033
5034     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5035     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5036                                      SUBREG_BYTE (operands[4]));
5037     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5038     DONE;
5039   }
5040 )
5041
5042 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5043   [(set (match_operand:GPI 0 "register_operand" "=&r")
5044         (ashift:GPI
5045           (match_operand:GPI 1 "register_operand" "r")
5046           (minus:QI (match_operand 2 "const_int_operand" "n")
5047                     (match_operator 5 "subreg_lowpart_operator"
5048                     [(and:SI (match_operand:SI 3 "register_operand" "r")
5049                              (match_operand 4 "const_int_operand" "n"))]))))]
5050   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5051    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5052   "#"
5053   "&& true"
5054   [(const_int 0)]
5055   {
5056     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5057                : operands[0]);
5058
5059     emit_insn (gen_negsi2 (tmp, operands[3]));
5060
5061     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5062     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5063                                      SUBREG_BYTE (operands[5]));
5064
5065     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5066     DONE;
5067   }
5068 )
5069
5070 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5071   [(set (match_operand:DI 0 "register_operand" "=r")
5072         (SHIFT:DI
5073           (match_operand:DI 1 "register_operand" "r")
5074           (match_operator 4 "subreg_lowpart_operator"
5075            [(and:SI (match_operand:SI 2 "register_operand" "r")
5076                     (match_operand 3 "const_int_operand" "n"))])))]
5077   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5078 {
5079   rtx xop[3];
5080   xop[0] = operands[0];
5081   xop[1] = operands[1];
5082   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5083   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5084   return "";
5085 }
5086   [(set_attr "type" "shift_reg")]
5087 )
5088
5089 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5090   [(set (match_operand:GPI 0 "register_operand" "=&r")
5091         (ASHIFT:GPI
5092           (match_operand:GPI 1 "register_operand" "r")
5093           (minus:QI (match_operand 2 "const_int_operand" "n")
5094                     (match_operand:QI 3 "register_operand" "r"))))]
5095   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5096   "#"
5097   "&& true"
5098   [(const_int 0)]
5099   {
5100     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5101
5102     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5103                : gen_lowpart (SImode, operands[0]));
5104
5105     emit_insn (gen_negsi2 (tmp, subreg_tmp));
5106
5107     rtx and_op = gen_rtx_AND (SImode, tmp,
5108                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5109
5110     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5111
5112     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5113     DONE;
5114   }
5115   [(set_attr "length" "8")]
5116 )
5117
5118 ;; Logical left shift using SISD or Integer instruction
5119 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5120   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5121         (ashift:GPI
5122           (match_operand:GPI 1 "register_operand" "r,r,w,w")
5123           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5124   ""
5125   "@
5126    lsl\t%<w>0, %<w>1, %2
5127    lsl\t%<w>0, %<w>1, %<w>2
5128    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5129    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5130   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5131    (set_attr "arch" "*,*,simd,simd")]
5132 )
5133
5134 ;; Logical right shift using SISD or Integer instruction
5135 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5136   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5137         (lshiftrt:GPI
5138          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5139          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5140                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5141   ""
5142   "@
5143    lsr\t%<w>0, %<w>1, %2
5144    lsr\t%<w>0, %<w>1, %<w>2
5145    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5146    #
5147    #"
5148   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5149    (set_attr "arch" "*,*,simd,simd,simd")]
5150 )
5151
5152 (define_split
5153   [(set (match_operand:DI 0 "aarch64_simd_register")
5154         (lshiftrt:DI
5155            (match_operand:DI 1 "aarch64_simd_register")
5156            (match_operand:QI 2 "aarch64_simd_register")))]
5157   "TARGET_SIMD && reload_completed"
5158   [(set (match_dup 3)
5159         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5160    (set (match_dup 0)
5161         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5162   {
5163     operands[3] = gen_lowpart (QImode, operands[0]);
5164   }
5165 )
5166
5167 (define_split
5168   [(set (match_operand:SI 0 "aarch64_simd_register")
5169         (lshiftrt:SI
5170            (match_operand:SI 1 "aarch64_simd_register")
5171            (match_operand:QI 2 "aarch64_simd_register")))]
5172   "TARGET_SIMD && reload_completed"
5173   [(set (match_dup 3)
5174         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5175    (set (match_dup 0)
5176         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5177   {
5178     operands[3] = gen_lowpart (QImode, operands[0]);
5179   }
5180 )
5181
5182 ;; Arithmetic right shift using SISD or Integer instruction
5183 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5184   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5185         (ashiftrt:GPI
5186           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5187           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5188                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5189   ""
5190   "@
5191    asr\t%<w>0, %<w>1, %2
5192    asr\t%<w>0, %<w>1, %<w>2
5193    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5194    #
5195    #"
5196   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5197    (set_attr "arch" "*,*,simd,simd,simd")]
5198 )
5199
5200 (define_split
5201   [(set (match_operand:DI 0 "aarch64_simd_register")
5202         (ashiftrt:DI
5203            (match_operand:DI 1 "aarch64_simd_register")
5204            (match_operand:QI 2 "aarch64_simd_register")))]
5205   "TARGET_SIMD && reload_completed"
5206   [(set (match_dup 3)
5207         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5208    (set (match_dup 0)
5209         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5210 {
5211   operands[3] = gen_lowpart (QImode, operands[0]);
5212 }
5213 )
5214
5215 (define_split
5216   [(set (match_operand:SI 0 "aarch64_simd_register")
5217         (ashiftrt:SI
5218            (match_operand:SI 1 "aarch64_simd_register")
5219            (match_operand:QI 2 "aarch64_simd_register")))]
5220   "TARGET_SIMD && reload_completed"
5221   [(set (match_dup 3)
5222         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5223    (set (match_dup 0)
5224         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5225 {
5226   operands[3] = gen_lowpart (QImode, operands[0]);
5227 }
5228 )
5229
5230 (define_insn "*aarch64_sisd_ushl"
5231   [(set (match_operand:DI 0 "register_operand" "=w")
5232         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5233                     (match_operand:QI 2 "register_operand" "w")]
5234                    UNSPEC_SISD_USHL))]
5235   "TARGET_SIMD"
5236   "ushl\t%d0, %d1, %d2"
5237   [(set_attr "type" "neon_shift_reg")]
5238 )
5239
5240 (define_insn "*aarch64_ushl_2s"
5241   [(set (match_operand:SI 0 "register_operand" "=w")
5242         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5243                     (match_operand:QI 2 "register_operand" "w")]
5244                    UNSPEC_USHL_2S))]
5245   "TARGET_SIMD"
5246   "ushl\t%0.2s, %1.2s, %2.2s"
5247   [(set_attr "type" "neon_shift_reg")]
5248 )
5249
5250 (define_insn "*aarch64_sisd_sshl"
5251   [(set (match_operand:DI 0 "register_operand" "=w")
5252         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5253                     (match_operand:QI 2 "register_operand" "w")]
5254                    UNSPEC_SISD_SSHL))]
5255   "TARGET_SIMD"
5256   "sshl\t%d0, %d1, %d2"
5257   [(set_attr "type" "neon_shift_reg")]
5258 )
5259
5260 (define_insn "*aarch64_sshl_2s"
5261   [(set (match_operand:SI 0 "register_operand" "=w")
5262         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5263                     (match_operand:QI 2 "register_operand" "w")]
5264                    UNSPEC_SSHL_2S))]
5265   "TARGET_SIMD"
5266   "sshl\t%0.2s, %1.2s, %2.2s"
5267   [(set_attr "type" "neon_shift_reg")]
5268 )
5269
5270 (define_insn "*aarch64_sisd_neg_qi"
5271   [(set (match_operand:QI 0 "register_operand" "=w")
5272         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5273                    UNSPEC_SISD_NEG))]
5274   "TARGET_SIMD"
5275   "neg\t%d0, %d1"
5276   [(set_attr "type" "neon_neg")]
5277 )
5278
5279 ;; Rotate right
5280 (define_insn "*ror<mode>3_insn"
5281   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5282      (rotatert:GPI
5283        (match_operand:GPI 1 "register_operand" "r,r")
5284        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5285   ""
5286   "@
5287    ror\\t%<w>0, %<w>1, %2
5288    ror\\t%<w>0, %<w>1, %<w>2"
5289   [(set_attr "type" "rotate_imm,shift_reg")]
5290 )
5291
5292 ;; zero_extend version of above
5293 (define_insn "*<optab>si3_insn_uxtw"
5294   [(set (match_operand:DI 0 "register_operand" "=r,r")
5295         (zero_extend:DI (SHIFT:SI
5296          (match_operand:SI 1 "register_operand" "r,r")
5297          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5298   ""
5299   "@
5300    <shift>\\t%w0, %w1, %2
5301    <shift>\\t%w0, %w1, %w2"
5302   [(set_attr "type" "bfx,shift_reg")]
5303 )
5304
5305 (define_insn "*<optab><mode>3_insn"
5306   [(set (match_operand:SHORT 0 "register_operand" "=r")
5307         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5308                       (match_operand 2 "const_int_operand" "n")))]
5309   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5310 {
5311   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5312   return "<bfshift>\t%w0, %w1, %2, %3";
5313 }
5314   [(set_attr "type" "bfx")]
5315 )
5316
5317 (define_insn "*extr<mode>5_insn"
5318   [(set (match_operand:GPI 0 "register_operand" "=r")
5319         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5320                              (match_operand 3 "const_int_operand" "n"))
5321                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5322                                (match_operand 4 "const_int_operand" "n"))))]
5323   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5324    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5325   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5326   [(set_attr "type" "rotate_imm")]
5327 )
5328
5329 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5330 ;; so we have to match both orderings.
5331 (define_insn "*extr<mode>5_insn_alt"
5332   [(set (match_operand:GPI 0 "register_operand" "=r")
5333         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5334                                 (match_operand 4 "const_int_operand" "n"))
5335                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5336                               (match_operand 3 "const_int_operand" "n"))))]
5337   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5338    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5339        == GET_MODE_BITSIZE (<MODE>mode))"
5340   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5341   [(set_attr "type" "rotate_imm")]
5342 )
5343
5344 ;; zero_extend version of the above
5345 (define_insn "*extrsi5_insn_uxtw"
5346   [(set (match_operand:DI 0 "register_operand" "=r")
5347         (zero_extend:DI
5348          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5349                             (match_operand 3 "const_int_operand" "n"))
5350                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5351                               (match_operand 4 "const_int_operand" "n")))))]
5352   "UINTVAL (operands[3]) < 32 &&
5353    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5354   "extr\\t%w0, %w1, %w2, %4"
5355   [(set_attr "type" "rotate_imm")]
5356 )
5357
5358 (define_insn "*extrsi5_insn_uxtw_alt"
5359   [(set (match_operand:DI 0 "register_operand" "=r")
5360         (zero_extend:DI
5361          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5362                                (match_operand 4 "const_int_operand" "n"))
5363                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5364                             (match_operand 3 "const_int_operand" "n")))))]
5365   "UINTVAL (operands[3]) < 32 &&
5366    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5367   "extr\\t%w0, %w1, %w2, %4"
5368   [(set_attr "type" "rotate_imm")]
5369 )
5370
5371 (define_insn "*ror<mode>3_insn"
5372   [(set (match_operand:GPI 0 "register_operand" "=r")
5373         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5374                     (match_operand 2 "const_int_operand" "n")))]
5375   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5376 {
5377   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5378   return "ror\\t%<w>0, %<w>1, %3";
5379 }
5380   [(set_attr "type" "rotate_imm")]
5381 )
5382
5383 ;; zero_extend version of the above
5384 (define_insn "*rorsi3_insn_uxtw"
5385   [(set (match_operand:DI 0 "register_operand" "=r")
5386         (zero_extend:DI
5387          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5388                     (match_operand 2 "const_int_operand" "n"))))]
5389   "UINTVAL (operands[2]) < 32"
5390 {
5391   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5392   return "ror\\t%w0, %w1, %3";
5393 }
5394   [(set_attr "type" "rotate_imm")]
5395 )
5396
5397 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5398   [(set (match_operand:GPI 0 "register_operand" "=r")
5399         (ANY_EXTEND:GPI
5400          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5401                        (match_operand 2 "const_int_operand" "n"))))]
5402   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5403 {
5404   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5405   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5406 }
5407   [(set_attr "type" "bfx")]
5408 )
5409
5410 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5411   [(set (match_operand:GPI 0 "register_operand" "=r")
5412         (zero_extend:GPI
5413          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5414                          (match_operand 2 "const_int_operand" "n"))))]
5415   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5416 {
5417   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5418   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5419 }
5420   [(set_attr "type" "bfx")]
5421 )
5422
5423 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5424   [(set (match_operand:GPI 0 "register_operand" "=r")
5425         (sign_extend:GPI
5426          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5427                          (match_operand 2 "const_int_operand" "n"))))]
5428   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5429 {
5430   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5431   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5432 }
5433   [(set_attr "type" "bfx")]
5434 )
5435
5436 ;; -------------------------------------------------------------------
5437 ;; Bitfields
5438 ;; -------------------------------------------------------------------
5439
5440 (define_expand "<optab>"
5441   [(set (match_operand:DI 0 "register_operand")
5442         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5443                         (match_operand 2
5444                           "aarch64_simd_shift_imm_offset_di")
5445                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5446   ""
5447   {
5448     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5449                    1, GET_MODE_BITSIZE (DImode) - 1))
5450      FAIL;
5451   }
5452 )
5453
5454
5455 (define_insn "*<optab><mode>"
5456   [(set (match_operand:GPI 0 "register_operand" "=r")
5457         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5458                          (match_operand 2
5459                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5460                          (match_operand 3
5461                            "aarch64_simd_shift_imm_<mode>" "n")))]
5462   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5463              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5464   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5465   [(set_attr "type" "bfx")]
5466 )
5467
5468 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5469 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5470 (define_split
5471   [(set (match_operand:DI 0 "register_operand")
5472         (zero_extract:DI (match_operand:DI 1 "register_operand")
5473                          (match_operand 2
5474                            "aarch64_simd_shift_imm_offset_di")
5475                          (match_operand 3
5476                            "aarch64_simd_shift_imm_di")))]
5477   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5478              GET_MODE_BITSIZE (DImode) - 1)
5479    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5480        == GET_MODE_BITSIZE (SImode)"
5481   [(set (match_dup 0)
5482         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5483   {
5484     operands[4] = gen_lowpart (SImode, operands[1]);
5485   }
5486 )
5487
5488 ;; Bitfield Insert (insv)
5489 (define_expand "insv<mode>"
5490   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5491                           (match_operand 1 "const_int_operand")
5492                           (match_operand 2 "const_int_operand"))
5493         (match_operand:GPI 3 "general_operand"))]
5494   ""
5495 {
5496   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5497   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5498   rtx value = operands[3];
5499
5500   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5501     FAIL;
5502
5503   if (CONST_INT_P (value))
5504     {
5505       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5506
5507       /* Prefer AND/OR for inserting all zeros or all ones.  */
5508       if ((UINTVAL (value) & mask) == 0
5509            || (UINTVAL (value) & mask) == mask)
5510         FAIL;
5511
5512       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5513       if (width == 16 && (pos % 16) == 0)
5514         DONE;
5515     }
5516   operands[3] = force_reg (<MODE>mode, value);
5517 })
5518
5519 (define_insn "*insv_reg<mode>"
5520   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5521                           (match_operand 1 "const_int_operand" "n")
5522                           (match_operand 2 "const_int_operand" "n"))
5523         (match_operand:GPI 3 "register_operand" "r"))]
5524   "!(UINTVAL (operands[1]) == 0
5525      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5526          > GET_MODE_BITSIZE (<MODE>mode)))"
5527   "bfi\\t%<w>0, %<w>3, %2, %1"
5528   [(set_attr "type" "bfm")]
5529 )
5530
5531 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5532   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5533                           (match_operand 1 "const_int_operand" "n")
5534                           (match_operand 2 "const_int_operand" "n"))
5535         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5536   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5537   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5538   [(set_attr "type" "bfm")]
5539 )
5540
5541 ;;  Match a bfi instruction where the shift of OP3 means that we are
5542 ;;  actually copying the least significant bits of OP3 into OP0 by way
5543 ;;  of the AND masks and the IOR instruction.  A similar instruction
5544 ;;  with the two parts of the IOR swapped around was never triggered
5545 ;;  in a bootstrap build and test of GCC so it was not included.
5546
5547 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5548   [(set (match_operand:GPI 0 "register_operand" "=r")
5549         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5550                           (match_operand:GPI 2 "const_int_operand" "n"))
5551                  (and:GPI (ashift:GPI
5552                            (match_operand:GPI 3 "register_operand" "r")
5553                            (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5554                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5555   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5556                                       UINTVAL (operands[4]),
5557                                       UINTVAL(operands[5]))"
5558   "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5559   [(set_attr "type" "bfm")]
5560 )
5561
5562 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5563   [(set (match_operand:GPI 0 "register_operand" "=r")
5564         (ior:GPI (and:GPI (ashift:GPI
5565                            (match_operand:GPI 1 "register_operand" "r")
5566                            (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5567                           (match_operand:GPI 3 "const_int_operand" "n"))
5568                  (and:GPI (match_operand:GPI 4 "register_operand" "0")
5569                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5570   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5571                                       UINTVAL (operands[2]),
5572                                       UINTVAL(operands[3]))"
5573   "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5574   [(set_attr "type" "bfm")]
5575 )
5576
5577 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5578 ;; the shift is large enough to remove the need for an AND instruction.
5579
5580 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5581   [(set (match_operand:GPI 0 "register_operand" "=r")
5582         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5583                           (match_operand:GPI 2 "const_int_operand" "n"))
5584                  (ashift:GPI
5585                           (match_operand:GPI 3 "register_operand" "r")
5586                           (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5587   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5588                                       UINTVAL (operands[4]),
5589                                       HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5590 {
5591   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5592   return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5593 }
5594   [(set_attr "type" "bfm")]
5595 )
5596
5597 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5598   [(set (match_operand:GPI 0 "register_operand" "=r")
5599         (ior:GPI (ashift:GPI
5600                           (match_operand:GPI 1 "register_operand" "r")
5601                           (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5602                  (and:GPI (match_operand:GPI 3 "register_operand" "0")
5603                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5604   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5605                                       UINTVAL (operands[2]),
5606                                       HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5607 {
5608   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5609   return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5610 }
5611   [(set_attr "type" "bfm")]
5612 )
5613
5614 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5615 ;; copying the least significant bits of OP3 to OP0.  We need two versions
5616 ;; of the instruction to handle different checks on the constant values.
5617
5618 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5619   [(set (match_operand:GPI 0 "register_operand" "=r")
5620         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5621                           (match_operand:GPI 2 "const_int_operand" "n"))
5622                  (and:GPI (match_operand:GPI 3 "register_operand" "r")
5623                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5624   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5625                                       UINTVAL (operands[4]))"
5626   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5627   [(set_attr "type" "bfm")]
5628 )
5629
5630 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5631   [(set (match_operand:GPI 0 "register_operand" "=r")
5632         (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5633                           (match_operand:GPI 4 "const_int_operand" "n"))
5634                  (and:GPI (match_operand:GPI 1 "register_operand" "0")
5635                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5636   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5637                                       UINTVAL (operands[4]))"
5638   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5639   [(set_attr "type" "bfm")]
5640 )
5641
5642 (define_insn "*extr_insv_lower_reg<mode>"
5643   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5644                           (match_operand 1 "const_int_operand" "n")
5645                           (const_int 0))
5646         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5647                           (match_dup 1)
5648                           (match_operand 3 "const_int_operand" "n")))]
5649   "!(UINTVAL (operands[1]) == 0
5650      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5651          > GET_MODE_BITSIZE (<MODE>mode)))"
5652   "bfxil\\t%<w>0, %<w>2, %3, %1"
5653   [(set_attr "type" "bfm")]
5654 )
5655
5656 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5657   [(set (match_operand:GPI 0 "register_operand" "=r")
5658         (ashift:GPI (ANY_EXTEND:GPI
5659                      (match_operand:ALLX 1 "register_operand" "r"))
5660                     (match_operand 2 "const_int_operand" "n")))]
5661   "UINTVAL (operands[2]) < <GPI:sizen>"
5662 {
5663   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5664               ? GEN_INT (<ALLX:sizen>)
5665               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5666   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5667 }
5668   [(set_attr "type" "bfx")]
5669 )
5670
5671 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5672
5673 (define_insn "*andim_ashift<mode>_bfiz"
5674   [(set (match_operand:GPI 0 "register_operand" "=r")
5675         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5676                              (match_operand 2 "const_int_operand" "n"))
5677                  (match_operand 3 "const_int_operand" "n")))]
5678   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5679   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5680   [(set_attr "type" "bfx")]
5681 )
5682
5683 ;; Match sbfiz pattern in a shift left + shift right operation.
5684
5685 (define_insn "*ashift<mode>_extv_bfiz"
5686   [(set (match_operand:GPI 0 "register_operand" "=r")
5687         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5688                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5689                                       (const_int 0))
5690                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5691   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5692              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5693   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5694   [(set_attr "type" "bfx")]
5695 )
5696
5697 ;; When the bit position and width of the equivalent extraction add up to 32
5698 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5699 ;; zero-extension of the X-reg.
5700 (define_split
5701   [(set (match_operand:DI 0 "register_operand")
5702         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5703                              (match_operand 2 "const_int_operand"))
5704                  (match_operand 3 "const_int_operand")))]
5705  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5706   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5707       == GET_MODE_BITSIZE (SImode)"
5708   [(set (match_dup 0)
5709         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5710   {
5711     operands[4] = gen_lowpart (SImode, operands[1]);
5712   }
5713 )
5714
5715 (define_insn "bswap<mode>2"
5716   [(set (match_operand:GPI 0 "register_operand" "=r")
5717         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5718   ""
5719   "rev\\t%<w>0, %<w>1"
5720   [(set_attr "type" "rev")]
5721 )
5722
5723 (define_insn "bswaphi2"
5724   [(set (match_operand:HI 0 "register_operand" "=r")
5725         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5726   ""
5727   "rev16\\t%w0, %w1"
5728   [(set_attr "type" "rev")]
5729 )
5730
5731 (define_insn "*aarch64_bfxil<mode>"
5732   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5733     (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5734                     (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5735             (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5736                     (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5737   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5738   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5739     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5740   {
5741     switch (which_alternative)
5742     {
5743       case 0:
5744         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5745         return "bfxil\\t%<w>0, %<w>1, 0, %3";
5746       case 1:
5747         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5748         return "bfxil\\t%<w>0, %<w>2, 0, %3";
5749       default:
5750         gcc_unreachable ();
5751     }
5752   }
5753   [(set_attr "type" "bfm")]
5754 )
5755
5756 ; Zero-extended version of above (aarch64_bfxil)
5757 (define_insn "*aarch64_bfxilsi_uxtw"
5758   [(set (match_operand:DI 0 "register_operand" "=r,r")
5759         (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5760                                         "r,0")
5761                     (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5762             (and:SI (match_operand:SI 2 "register_operand" "0,r")
5763                     (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5764   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5765   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5766     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5767   {
5768     switch (which_alternative)
5769     {
5770       case 0:
5771         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5772         return "bfxil\\t%0, %1, 0, %3";
5773       case 1:
5774         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5775         return "bfxil\\t%0, %2, 0, %3";
5776       default:
5777         gcc_unreachable ();
5778     }
5779   }
5780   [(set_attr "type" "bfm")]
5781 )
5782
5783 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5784 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5785 ;; each valid permutation.
5786
5787 (define_insn "rev16<mode>2"
5788   [(set (match_operand:GPI 0 "register_operand" "=r")
5789         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5790                                       (const_int 8))
5791                           (match_operand:GPI 3 "const_int_operand" "n"))
5792                  (and:GPI (lshiftrt:GPI (match_dup 1)
5793                                         (const_int 8))
5794                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5795   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5796    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5797   "rev16\\t%<w>0, %<w>1"
5798   [(set_attr "type" "rev")]
5799 )
5800
5801 (define_insn "rev16<mode>2_alt"
5802   [(set (match_operand:GPI 0 "register_operand" "=r")
5803         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5804                                         (const_int 8))
5805                           (match_operand:GPI 2 "const_int_operand" "n"))
5806                  (and:GPI (ashift:GPI (match_dup 1)
5807                                       (const_int 8))
5808                           (match_operand:GPI 3 "const_int_operand" "n"))))]
5809   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5810    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5811   "rev16\\t%<w>0, %<w>1"
5812   [(set_attr "type" "rev")]
5813 )
5814
5815 ;; zero_extend version of above
5816 (define_insn "*bswapsi2_uxtw"
5817   [(set (match_operand:DI 0 "register_operand" "=r")
5818         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5819   ""
5820   "rev\\t%w0, %w1"
5821   [(set_attr "type" "rev")]
5822 )
5823
5824 ;; -------------------------------------------------------------------
5825 ;; Floating-point intrinsics
5826 ;; -------------------------------------------------------------------
5827
5828 ;; frint floating-point round to integral standard patterns.
5829 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5830
5831 (define_insn "<frint_pattern><mode>2"
5832   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5833         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5834          FRINT))]
5835   "TARGET_FLOAT"
5836   "frint<frint_suffix>\\t%<s>0, %<s>1"
5837   [(set_attr "type" "f_rint<stype>")]
5838 )
5839
5840 ;; frcvt floating-point round to integer and convert standard patterns.
5841 ;; Expands to lbtrunc, lceil, lfloor, lround.
5842 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5843   [(set (match_operand:GPI 0 "register_operand" "=r")
5844         (FIXUORS:GPI
5845           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5846            FCVT)))]
5847   "TARGET_FLOAT"
5848   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5849   [(set_attr "type" "f_cvtf2i")]
5850 )
5851
5852 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5853   [(set (match_operand:GPI 0 "register_operand" "=r")
5854         (FIXUORS:GPI
5855           (mult:GPF
5856             (match_operand:GPF 1 "register_operand" "w")
5857             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5858   "TARGET_FLOAT
5859    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5860                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5861   {
5862     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5863     char buf[64];
5864     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5865     output_asm_insn (buf, operands);
5866     return "";
5867   }
5868   [(set_attr "type" "f_cvtf2i")]
5869 )
5870
5871 ;; fma - expand fma into patterns with the accumulator operand first since
5872 ;; reusing the accumulator results in better register allocation.
5873 ;; The register allocator considers copy preferences in operand order,
5874 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5875
5876 (define_expand "fma<mode>4"
5877   [(set (match_operand:GPF_F16 0 "register_operand")
5878         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5879                      (match_operand:GPF_F16 2 "register_operand")
5880                      (match_operand:GPF_F16 3 "register_operand")))]
5881   "TARGET_FLOAT"
5882 )
5883
5884 (define_insn "*aarch64_fma<mode>4"
5885   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5886         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5887                      (match_operand:GPF_F16 3 "register_operand" "w")
5888                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5889   "TARGET_FLOAT"
5890   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5891   [(set_attr "type" "fmac<stype>")]
5892 )
5893
5894 (define_expand "fnma<mode>4"
5895   [(set (match_operand:GPF_F16 0 "register_operand")
5896         (fma:GPF_F16
5897           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5898           (match_operand:GPF_F16 2 "register_operand")
5899           (match_operand:GPF_F16 3 "register_operand")))]
5900   "TARGET_FLOAT"
5901 )
5902
5903 (define_insn "*aarch64_fnma<mode>4"
5904   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5905         (fma:GPF_F16
5906           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5907           (match_operand:GPF_F16 3 "register_operand" "w")
5908           (match_operand:GPF_F16 1 "register_operand" "w")))]
5909   "TARGET_FLOAT"
5910   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5911   [(set_attr "type" "fmac<stype>")]
5912 )
5913
5914
5915 (define_expand "fms<mode>4"
5916   [(set (match_operand:GPF 0 "register_operand")
5917         (fma:GPF (match_operand:GPF 1 "register_operand")
5918                  (match_operand:GPF 2 "register_operand")
5919                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5920   "TARGET_FLOAT"
5921 )
5922
5923 (define_insn "*aarch64_fms<mode>4"
5924   [(set (match_operand:GPF 0 "register_operand" "=w")
5925         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5926                  (match_operand:GPF 3 "register_operand" "w")
5927                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5928   "TARGET_FLOAT"
5929   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5930   [(set_attr "type" "fmac<s>")]
5931 )
5932
5933 (define_expand "fnms<mode>4"
5934   [(set (match_operand:GPF 0 "register_operand")
5935         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5936                  (match_operand:GPF 2 "register_operand")
5937                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5938   "TARGET_FLOAT"
5939 )
5940
5941 (define_insn "*aarch64_fnms<mode>4"
5942   [(set (match_operand:GPF 0 "register_operand" "=w")
5943         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5944                  (match_operand:GPF 3 "register_operand" "w")
5945                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5946   "TARGET_FLOAT"
5947   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5948   [(set_attr "type" "fmac<s>")]
5949 )
5950
5951 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5952 (define_insn "*aarch64_fnmadd<mode>4"
5953   [(set (match_operand:GPF 0 "register_operand" "=w")
5954         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5955                           (match_operand:GPF 3 "register_operand" "w")
5956                           (match_operand:GPF 1 "register_operand" "w"))))]
5957   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5958   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5959   [(set_attr "type" "fmac<s>")]
5960 )
5961
5962 ;; -------------------------------------------------------------------
5963 ;; Floating-point conversions
5964 ;; -------------------------------------------------------------------
5965
5966 (define_insn "extendsfdf2"
5967   [(set (match_operand:DF 0 "register_operand" "=w")
5968         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5969   "TARGET_FLOAT"
5970   "fcvt\\t%d0, %s1"
5971   [(set_attr "type" "f_cvt")]
5972 )
5973
5974 (define_insn "extendhfsf2"
5975   [(set (match_operand:SF 0 "register_operand" "=w")
5976         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5977   "TARGET_FLOAT"
5978   "fcvt\\t%s0, %h1"
5979   [(set_attr "type" "f_cvt")]
5980 )
5981
5982 (define_insn "extendhfdf2"
5983   [(set (match_operand:DF 0 "register_operand" "=w")
5984         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5985   "TARGET_FLOAT"
5986   "fcvt\\t%d0, %h1"
5987   [(set_attr "type" "f_cvt")]
5988 )
5989
5990 (define_insn "truncdfsf2"
5991   [(set (match_operand:SF 0 "register_operand" "=w")
5992         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5993   "TARGET_FLOAT"
5994   "fcvt\\t%s0, %d1"
5995   [(set_attr "type" "f_cvt")]
5996 )
5997
5998 (define_insn "truncsfhf2"
5999   [(set (match_operand:HF 0 "register_operand" "=w")
6000         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6001   "TARGET_FLOAT"
6002   "fcvt\\t%h0, %s1"
6003   [(set_attr "type" "f_cvt")]
6004 )
6005
6006 (define_insn "truncdfhf2"
6007   [(set (match_operand:HF 0 "register_operand" "=w")
6008         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6009   "TARGET_FLOAT"
6010   "fcvt\\t%h0, %d1"
6011   [(set_attr "type" "f_cvt")]
6012 )
6013
6014 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6015 ;; and making r = w more expensive
6016
6017 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
6018   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
6019         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6020   "TARGET_FLOAT"
6021   "@
6022    fcvtz<su>\t%<s>0, %<s>1
6023    fcvtz<su>\t%<w>0, %<s>1"
6024   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
6025 )
6026
6027 ;; Convert HF -> SI or DI
6028
6029 (define_insn "<optab>_trunchf<GPI:mode>2"
6030   [(set (match_operand:GPI 0 "register_operand" "=r")
6031         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6032   "TARGET_FP_F16INST"
6033   "fcvtz<su>\t%<w>0, %h1"
6034   [(set_attr "type" "f_cvtf2i")]
6035 )
6036
6037 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6038 ;; input in a fp register and output in a integer register
6039
6040 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6041   [(set (match_operand:GPI 0 "register_operand" "=r")
6042         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6043   "TARGET_FLOAT"
6044   "fcvtz<su>\t%<w>0, %<fpw>1"
6045   [(set_attr "type" "f_cvtf2i")]
6046 )
6047
6048 (define_insn "*fix_to_zero_extend<mode>di2"
6049   [(set (match_operand:DI 0 "register_operand" "=r")
6050         (zero_extend:DI
6051          (unsigned_fix:SI
6052           (match_operand:GPF 1 "register_operand" "w"))))]
6053   "TARGET_FLOAT"
6054   "fcvtzu\t%w0, %<s>1"
6055   [(set_attr "type" "f_cvtf2i")]
6056 )
6057
6058 ;; Equal width integer to fp and multiply combine.
6059 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6060   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6061         (mult:GPF (FLOATUORS:GPF
6062                    (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6063                    (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6064   "TARGET_FLOAT"
6065   {
6066     operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6067     switch (which_alternative)
6068     {
6069       case 0:
6070         return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6071       case 1:
6072         return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6073       default:
6074         gcc_unreachable ();
6075     }
6076   }
6077   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6078    (set_attr "arch" "simd,fp")]
6079 )
6080
6081 ;; Unequal width integer to fp and multiply combine.
6082 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6083   [(set (match_operand:GPF 0 "register_operand" "=w")
6084         (mult:GPF (FLOATUORS:GPF
6085                    (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6086                    (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6087   "TARGET_FLOAT"
6088   {
6089     operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6090     return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6091   }
6092   [(set_attr "type" "f_cvti2f")]
6093 )
6094
6095 ;; Equal width integer to fp conversion.
6096 (define_insn "<optab><fcvt_target><GPF:mode>2"
6097   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6098         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6099   "TARGET_FLOAT"
6100   "@
6101    <su_optab>cvtf\t%<GPF:s>0, %<s>1
6102    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6103   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6104    (set_attr "arch" "simd,fp")]
6105 )
6106
6107 ;; Unequal width integer to fp conversions.
6108 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6109   [(set (match_operand:GPF 0 "register_operand" "=w")
6110         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6111   "TARGET_FLOAT"
6112   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6113   [(set_attr "type" "f_cvti2f")]
6114 )
6115
6116 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6117 ;; midend will arrange for an SImode conversion to HFmode to first go
6118 ;; through DFmode, then to HFmode.  But first it will try converting
6119 ;; to DImode then down, which would match our DImode pattern below and
6120 ;; give very poor code-generation.  So, we must provide our own emulation
6121 ;; of the mid-end logic.
6122
6123 (define_insn "aarch64_fp16_<optab><mode>hf2"
6124   [(set (match_operand:HF 0 "register_operand" "=w")
6125         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6126   "TARGET_FP_F16INST"
6127   "<su_optab>cvtf\t%h0, %<w>1"
6128   [(set_attr "type" "f_cvti2f")]
6129 )
6130
6131 (define_expand "<optab>sihf2"
6132   [(set (match_operand:HF 0 "register_operand")
6133         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6134   "TARGET_FLOAT"
6135 {
6136   if (TARGET_FP_F16INST)
6137     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6138   else
6139     {
6140       rtx convert_target = gen_reg_rtx (DFmode);
6141       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6142       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6143     }
6144   DONE;
6145 }
6146 )
6147
6148 ;; For DImode there is no wide enough floating-point mode that we
6149 ;; can convert through natively (TFmode would work, but requires a library
6150 ;; call).  However, we know that any value >= 65504 will be rounded
6151 ;; to infinity on conversion.  This is well within the range of SImode, so
6152 ;; we can:
6153 ;;   Saturate to SImode.
6154 ;;   Convert from that to DFmode
6155 ;;   Convert from that to HFmode (phew!).
6156 ;; Note that the saturation to SImode requires the SIMD extensions.  If
6157 ;; we ever need to provide this pattern where the SIMD extensions are not
6158 ;; available, we would need a different approach.
6159
6160 (define_expand "<optab>dihf2"
6161   [(set (match_operand:HF 0 "register_operand")
6162         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6163   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6164 {
6165   if (TARGET_FP_F16INST)
6166     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6167   else
6168     {
6169       rtx sat_target = gen_reg_rtx (SImode);
6170       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6171       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6172     }
6173
6174   DONE;
6175 }
6176 )
6177
6178 ;; Convert between fixed-point and floating-point (scalar modes)
6179
6180 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6181   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6182         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6183                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6184          FCVT_F2FIXED))]
6185   ""
6186   "@
6187    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6188    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6189   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6190    (set_attr "arch" "fp,simd")]
6191 )
6192
6193 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6194   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6195         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6196                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6197          FCVT_FIXED2F))]
6198   ""
6199   "@
6200    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6201    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6202   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6203    (set_attr "arch" "fp,simd")]
6204 )
6205
6206 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6207   [(set (match_operand:GPI 0 "register_operand" "=r")
6208         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6209                      (match_operand:SI 2 "immediate_operand" "i")]
6210          FCVT_F2FIXED))]
6211   "TARGET_FP_F16INST"
6212    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6213   [(set_attr "type" "f_cvtf2i")]
6214 )
6215
6216 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6217   [(set (match_operand:HF 0 "register_operand" "=w")
6218         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6219                     (match_operand:SI 2 "immediate_operand" "i")]
6220          FCVT_FIXED2F))]
6221   "TARGET_FP_F16INST"
6222   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6223   [(set_attr "type" "f_cvti2f")]
6224 )
6225
6226 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6227   [(set (match_operand:HI 0 "register_operand" "=w")
6228         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6229                     (match_operand:SI 2 "immediate_operand" "i")]
6230          FCVT_F2FIXED))]
6231   "TARGET_SIMD"
6232   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6233   [(set_attr "type" "neon_fp_to_int_s")]
6234 )
6235
6236 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6237   [(set (match_operand:HF 0 "register_operand" "=w")
6238         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6239                     (match_operand:SI 2 "immediate_operand" "i")]
6240          FCVT_FIXED2F))]
6241   "TARGET_SIMD"
6242   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6243   [(set_attr "type" "neon_int_to_fp_s")]
6244 )
6245
6246 ;; -------------------------------------------------------------------
6247 ;; Floating-point arithmetic
6248 ;; -------------------------------------------------------------------
6249
6250 (define_insn "add<mode>3"
6251   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6252         (plus:GPF_F16
6253          (match_operand:GPF_F16 1 "register_operand" "w")
6254          (match_operand:GPF_F16 2 "register_operand" "w")))]
6255   "TARGET_FLOAT"
6256   "fadd\\t%<s>0, %<s>1, %<s>2"
6257   [(set_attr "type" "fadd<stype>")]
6258 )
6259
6260 (define_insn "sub<mode>3"
6261   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6262         (minus:GPF_F16
6263          (match_operand:GPF_F16 1 "register_operand" "w")
6264          (match_operand:GPF_F16 2 "register_operand" "w")))]
6265   "TARGET_FLOAT"
6266   "fsub\\t%<s>0, %<s>1, %<s>2"
6267   [(set_attr "type" "fadd<stype>")]
6268 )
6269
6270 (define_insn "mul<mode>3"
6271   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6272         (mult:GPF_F16
6273          (match_operand:GPF_F16 1 "register_operand" "w")
6274          (match_operand:GPF_F16 2 "register_operand" "w")))]
6275   "TARGET_FLOAT"
6276   "fmul\\t%<s>0, %<s>1, %<s>2"
6277   [(set_attr "type" "fmul<stype>")]
6278 )
6279
6280 (define_insn "*fnmul<mode>3"
6281   [(set (match_operand:GPF 0 "register_operand" "=w")
6282         (mult:GPF
6283                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6284                  (match_operand:GPF 2 "register_operand" "w")))]
6285   "TARGET_FLOAT && !flag_rounding_math"
6286   "fnmul\\t%<s>0, %<s>1, %<s>2"
6287   [(set_attr "type" "fmul<s>")]
6288 )
6289
6290 (define_insn "*fnmul<mode>3"
6291   [(set (match_operand:GPF 0 "register_operand" "=w")
6292         (neg:GPF (mult:GPF
6293                  (match_operand:GPF 1 "register_operand" "w")
6294                  (match_operand:GPF 2 "register_operand" "w"))))]
6295   "TARGET_FLOAT"
6296   "fnmul\\t%<s>0, %<s>1, %<s>2"
6297   [(set_attr "type" "fmul<s>")]
6298 )
6299
6300 (define_expand "div<mode>3"
6301  [(set (match_operand:GPF_F16 0 "register_operand")
6302        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6303                     (match_operand:GPF_F16 2 "register_operand")))]
6304  "TARGET_FLOAT"
6305 {
6306   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6307     DONE;
6308
6309   operands[1] = force_reg (<MODE>mode, operands[1]);
6310 })
6311
6312 (define_insn "*div<mode>3"
6313   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6314         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6315                      (match_operand:GPF_F16 2 "register_operand" "w")))]
6316   "TARGET_FLOAT"
6317   "fdiv\\t%<s>0, %<s>1, %<s>2"
6318   [(set_attr "type" "fdiv<stype>")]
6319 )
6320
6321 (define_insn "neg<mode>2"
6322   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6323         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6324   "TARGET_FLOAT"
6325   "fneg\\t%<s>0, %<s>1"
6326   [(set_attr "type" "ffarith<stype>")]
6327 )
6328
6329 (define_expand "sqrt<mode>2"
6330   [(set (match_operand:GPF_F16 0 "register_operand")
6331         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6332   "TARGET_FLOAT"
6333 {
6334   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6335     DONE;
6336 })
6337
6338 (define_insn "*sqrt<mode>2"
6339   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6340         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6341   "TARGET_FLOAT"
6342   "fsqrt\\t%<s>0, %<s>1"
6343   [(set_attr "type" "fsqrt<stype>")]
6344 )
6345
6346 (define_insn "abs<mode>2"
6347   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6348         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6349   "TARGET_FLOAT"
6350   "fabs\\t%<s>0, %<s>1"
6351   [(set_attr "type" "ffarith<stype>")]
6352 )
6353
6354 ;; Given that smax/smin do not specify the result when either input is NaN,
6355 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6356 ;; for smin.
6357
6358 (define_insn "smax<mode>3"
6359   [(set (match_operand:GPF 0 "register_operand" "=w")
6360         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6361                   (match_operand:GPF 2 "register_operand" "w")))]
6362   "TARGET_FLOAT"
6363   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6364   [(set_attr "type" "f_minmax<s>")]
6365 )
6366
6367 (define_insn "smin<mode>3"
6368   [(set (match_operand:GPF 0 "register_operand" "=w")
6369         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6370                   (match_operand:GPF 2 "register_operand" "w")))]
6371   "TARGET_FLOAT"
6372   "fminnm\\t%<s>0, %<s>1, %<s>2"
6373   [(set_attr "type" "f_minmax<s>")]
6374 )
6375
6376 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6377 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6378 ;; which implement the IEEE fmax ()/fmin () functions.
6379 (define_insn "<maxmin_uns><mode>3"
6380   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6381         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6382                      (match_operand:GPF_F16 2 "register_operand" "w")]
6383                      FMAXMIN_UNS))]
6384   "TARGET_FLOAT"
6385   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6386   [(set_attr "type" "f_minmax<stype>")]
6387 )
6388
6389 (define_expand "lrint<GPF:mode><GPI:mode>2"
6390   [(match_operand:GPI 0 "register_operand")
6391    (match_operand:GPF 1 "register_operand")]
6392   "TARGET_FLOAT
6393    && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
6394    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6395 {
6396   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6397   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6398   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6399   DONE;
6400 }
6401 )
6402
6403 ;; For copysign (x, y), we want to generate:
6404 ;;
6405 ;;   LDR d2, #(1 << 63)
6406 ;;   BSL v2.8b, [y], [x]
6407 ;;
6408 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.  Because
6409 ;; we expect these operations to nearly always operate on
6410 ;; floating-point values, we do not want the operation to be
6411 ;; simplified into a bit-field insert operation that operates on the
6412 ;; integer side, since typically that would involve three inter-bank
6413 ;; register copies.  As we do not expect copysign to be followed by
6414 ;; other logical operations on the result, it seems preferable to keep
6415 ;; this as an unspec operation, rather than exposing the underlying
6416 ;; logic to the compiler.
6417
6418 (define_expand "copysign<GPF:mode>3"
6419   [(match_operand:GPF 0 "register_operand")
6420    (match_operand:GPF 1 "register_operand")
6421    (match_operand:GPF 2 "register_operand")]
6422   "TARGET_FLOAT && TARGET_SIMD"
6423 {
6424   rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6425   emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6426                                     << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6427   emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6428                                        bitmask));
6429   DONE;
6430 }
6431 )
6432
6433 (define_insn "copysign<GPF:mode>3_insn"
6434   [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6435         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6436                      (match_operand:GPF 2 "register_operand" "w,w,0,0")
6437                      (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6438          UNSPEC_COPYSIGN))]
6439   "TARGET_FLOAT && TARGET_SIMD"
6440   "@
6441    bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6442    bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6443    bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6444    bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6445   [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6446 )
6447
6448
6449 ;; For xorsign (x, y), we want to generate:
6450 ;;
6451 ;; LDR   d2, #1<<63
6452 ;; AND   v3.8B, v1.8B, v2.8B
6453 ;; EOR   v0.8B, v0.8B, v3.8B
6454 ;;
6455
6456 (define_expand "xorsign<mode>3"
6457   [(match_operand:GPF 0 "register_operand")
6458    (match_operand:GPF 1 "register_operand")
6459    (match_operand:GPF 2 "register_operand")]
6460   "TARGET_FLOAT && TARGET_SIMD"
6461 {
6462
6463   machine_mode imode = <V_INT_EQUIV>mode;
6464   rtx mask = gen_reg_rtx (imode);
6465   rtx op1x = gen_reg_rtx (imode);
6466   rtx op2x = gen_reg_rtx (imode);
6467
6468   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6469   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6470                                                      imode)));
6471
6472   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6473                                     lowpart_subreg (imode, operands[2],
6474                                                     <MODE>mode)));
6475   emit_insn (gen_xor<v_int_equiv>3 (op1x,
6476                                     lowpart_subreg (imode, operands[1],
6477                                                     <MODE>mode),
6478                                     op2x));
6479   emit_move_insn (operands[0],
6480                   lowpart_subreg (<MODE>mode, op1x, imode));
6481   DONE;
6482 }
6483 )
6484
6485 ;; -------------------------------------------------------------------
6486 ;; Reload support
6487 ;; -------------------------------------------------------------------
6488 ;; Reload Scalar Floating point modes from constant pool.
6489 ;; The AArch64 port doesn't have __int128 constant move support.
6490 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6491 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6492  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6493        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6494   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6495  "TARGET_FLOAT"
6496  {
6497    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6498    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6499    DONE;
6500  }
6501 )
6502
6503 ;; Reload Vector modes from constant pool.
6504 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6505  [(set (match_operand:VALL 0 "register_operand" "=w")
6506        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6507   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6508  "TARGET_FLOAT"
6509  {
6510    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6511    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6512    DONE;
6513  }
6514 )
6515
6516 (define_expand "@aarch64_reload_mov<mode>"
6517   [(set (match_operand:TX 0 "register_operand" "=w")
6518         (match_operand:TX 1 "register_operand" "w"))
6519    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6520   ]
6521   "TARGET_FLOAT"
6522   {
6523     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6524     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6525     gen_aarch64_movtilow_tilow (op0, op1);
6526     gen_aarch64_movdi_tihigh (operands[2], op1);
6527     gen_aarch64_movtihigh_di (op0, operands[2]);
6528     DONE;
6529   }
6530 )
6531
6532 ;; The following secondary reload helpers patterns are invoked
6533 ;; after or during reload as we don't want these patterns to start
6534 ;; kicking in during the combiner.
6535
6536 (define_insn "@aarch64_movdi_<mode>low"
6537   [(set (match_operand:DI 0 "register_operand" "=r")
6538         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6539                          (const_int 64) (const_int 0)))]
6540   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6541   "fmov\\t%x0, %d1"
6542   [(set_attr "type" "f_mrc")
6543    (set_attr "length" "4")
6544   ])
6545
6546 (define_insn "@aarch64_movdi_<mode>high"
6547   [(set (match_operand:DI 0 "register_operand" "=r")
6548         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6549                          (const_int 64) (const_int 64)))]
6550   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6551   "fmov\\t%x0, %1.d[1]"
6552   [(set_attr "type" "f_mrc")
6553    (set_attr "length" "4")
6554   ])
6555
6556 (define_insn "@aarch64_mov<mode>high_di"
6557   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6558                          (const_int 64) (const_int 64))
6559         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6560   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6561   "fmov\\t%0.d[1], %x1"
6562   [(set_attr "type" "f_mcr")
6563    (set_attr "length" "4")
6564   ])
6565
6566 (define_insn "@aarch64_mov<mode>low_di"
6567   [(set (match_operand:TX 0 "register_operand" "=w")
6568         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6569   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6570   "fmov\\t%d0, %x1"
6571   [(set_attr "type" "f_mcr")
6572    (set_attr "length" "4")
6573   ])
6574
6575 (define_insn "aarch64_movtilow_tilow"
6576   [(set (match_operand:TI 0 "register_operand" "=w")
6577         (zero_extend:TI
6578           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6579   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6580   "fmov\\t%d0, %d1"
6581   [(set_attr "type" "fmov")
6582    (set_attr "length" "4")
6583   ])
6584
6585 ;; There is a deliberate reason why the parameters of high and lo_sum's
6586 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6587 ;; and lo_sum's to be used with the labels defining the jump tables in
6588 ;; rodata section.
6589
6590 (define_expand "add_losym"
6591   [(set (match_operand 0 "register_operand")
6592         (lo_sum (match_operand 1 "register_operand")
6593                 (match_operand 2 "aarch64_valid_symref")))]
6594   ""
6595 {
6596   machine_mode mode = GET_MODE (operands[0]);
6597
6598   emit_insn ((mode == DImode
6599               ? gen_add_losym_di
6600               : gen_add_losym_si) (operands[0],
6601                                    operands[1],
6602                                    operands[2]));
6603   DONE;
6604 })
6605
6606 (define_insn "add_losym_<mode>"
6607   [(set (match_operand:P 0 "register_operand" "=r")
6608         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6609                   (match_operand 2 "aarch64_valid_symref" "S")))]
6610   ""
6611   "add\\t%<w>0, %<w>1, :lo12:%c2"
6612   [(set_attr "type" "alu_imm")]
6613 )
6614
6615 (define_insn "ldr_got_small_<mode>"
6616   [(set (match_operand:PTR 0 "register_operand" "=r")
6617         (unspec:PTR [(mem:PTR (lo_sum:PTR
6618                               (match_operand:PTR 1 "register_operand" "r")
6619                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6620                     UNSPEC_GOTSMALLPIC))]
6621   ""
6622   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6623   [(set_attr "type" "load_<ldst_sz>")]
6624 )
6625
6626 (define_insn "ldr_got_small_sidi"
6627   [(set (match_operand:DI 0 "register_operand" "=r")
6628         (zero_extend:DI
6629          (unspec:SI [(mem:SI (lo_sum:DI
6630                              (match_operand:DI 1 "register_operand" "r")
6631                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6632                     UNSPEC_GOTSMALLPIC)))]
6633   "TARGET_ILP32"
6634   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6635   [(set_attr "type" "load_4")]
6636 )
6637
6638 (define_insn "ldr_got_small_28k_<mode>"
6639   [(set (match_operand:PTR 0 "register_operand" "=r")
6640         (unspec:PTR [(mem:PTR (lo_sum:PTR
6641                               (match_operand:PTR 1 "register_operand" "r")
6642                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6643                     UNSPEC_GOTSMALLPIC28K))]
6644   ""
6645   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6646   [(set_attr "type" "load_<ldst_sz>")]
6647 )
6648
6649 (define_insn "ldr_got_small_28k_sidi"
6650   [(set (match_operand:DI 0 "register_operand" "=r")
6651         (zero_extend:DI
6652          (unspec:SI [(mem:SI (lo_sum:DI
6653                              (match_operand:DI 1 "register_operand" "r")
6654                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6655                     UNSPEC_GOTSMALLPIC28K)))]
6656   "TARGET_ILP32"
6657   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6658   [(set_attr "type" "load_4")]
6659 )
6660
6661 (define_insn "ldr_got_tiny"
6662   [(set (match_operand:DI 0 "register_operand" "=r")
6663         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6664                    UNSPEC_GOTTINYPIC))]
6665   ""
6666   "ldr\\t%0, %L1"
6667   [(set_attr "type" "load_8")]
6668 )
6669
6670 (define_insn "aarch64_load_tp_hard"
6671   [(set (match_operand:DI 0 "register_operand" "=r")
6672         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6673   ""
6674   "mrs\\t%0, tpidr_el0"
6675   [(set_attr "type" "mrs")]
6676 )
6677
6678 ;; The TLS ABI specifically requires that the compiler does not schedule
6679 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6680 ;; Therefore we treat the stubs as an atomic sequence.
6681 (define_expand "tlsgd_small_<mode>"
6682  [(parallel [(set (match_operand 0 "register_operand")
6683                   (call (mem:DI (match_dup 2)) (const_int 1)))
6684              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6685              (clobber (reg:DI LR_REGNUM))])]
6686  ""
6687 {
6688   operands[2] = aarch64_tls_get_addr ();
6689 })
6690
6691 (define_insn "*tlsgd_small_<mode>"
6692   [(set (match_operand 0 "register_operand" "")
6693         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6694    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6695    (clobber (reg:DI LR_REGNUM))
6696   ]
6697   ""
6698   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6699   [(set_attr "type" "call")
6700    (set_attr "length" "16")])
6701
6702 (define_insn "tlsie_small_<mode>"
6703   [(set (match_operand:PTR 0 "register_operand" "=r")
6704         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6705                    UNSPEC_GOTSMALLTLS))]
6706   ""
6707   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6708   [(set_attr "type" "load_4")
6709    (set_attr "length" "8")]
6710 )
6711
6712 (define_insn "tlsie_small_sidi"
6713   [(set (match_operand:DI 0 "register_operand" "=r")
6714         (zero_extend:DI
6715           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6716                       UNSPEC_GOTSMALLTLS)))]
6717   ""
6718   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6719   [(set_attr "type" "load_4")
6720    (set_attr "length" "8")]
6721 )
6722
6723 (define_insn "tlsie_tiny_<mode>"
6724   [(set (match_operand:PTR 0 "register_operand" "=&r")
6725         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6726                      (match_operand:PTR 2 "register_operand" "r")]
6727                    UNSPEC_GOTTINYTLS))]
6728   ""
6729   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6730   [(set_attr "type" "multiple")
6731    (set_attr "length" "8")]
6732 )
6733
6734 (define_insn "tlsie_tiny_sidi"
6735   [(set (match_operand:DI 0 "register_operand" "=&r")
6736         (zero_extend:DI
6737           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6738                       (match_operand:DI 2 "register_operand" "r")
6739                       ]
6740                       UNSPEC_GOTTINYTLS)))]
6741   ""
6742   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6743   [(set_attr "type" "multiple")
6744    (set_attr "length" "8")]
6745 )
6746
6747 (define_insn "tlsle12_<mode>"
6748   [(set (match_operand:P 0 "register_operand" "=r")
6749         (unspec:P [(match_operand:P 1 "register_operand" "r")
6750                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6751                    UNSPEC_TLSLE12))]
6752   ""
6753   "add\\t%<w>0, %<w>1, #%L2";
6754   [(set_attr "type" "alu_sreg")
6755    (set_attr "length" "4")]
6756 )
6757
6758 (define_insn "tlsle24_<mode>"
6759   [(set (match_operand:P 0 "register_operand" "=r")
6760         (unspec:P [(match_operand:P 1 "register_operand" "r")
6761                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6762                    UNSPEC_TLSLE24))]
6763   ""
6764   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6765   [(set_attr "type" "multiple")
6766    (set_attr "length" "8")]
6767 )
6768
6769 (define_insn "tlsle32_<mode>"
6770   [(set (match_operand:P 0 "register_operand" "=r")
6771         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6772                    UNSPEC_TLSLE32))]
6773   ""
6774   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6775   [(set_attr "type" "multiple")
6776    (set_attr "length" "8")]
6777 )
6778
6779 (define_insn "tlsle48_<mode>"
6780   [(set (match_operand:P 0 "register_operand" "=r")
6781         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6782                    UNSPEC_TLSLE48))]
6783   ""
6784   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6785   [(set_attr "type" "multiple")
6786    (set_attr "length" "12")]
6787 )
6788
6789 (define_expand "tlsdesc_small_<mode>"
6790   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6791   "TARGET_TLS_DESC"
6792   {
6793     if (TARGET_SVE)
6794       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6795     else
6796       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6797     DONE;
6798   }
6799 )
6800
6801 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6802 ;; R0 and LR.
6803 (define_insn "tlsdesc_small_advsimd_<mode>"
6804   [(set (reg:PTR R0_REGNUM)
6805         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6806                     UNSPEC_TLSDESC))
6807    (clobber (reg:DI LR_REGNUM))
6808    (clobber (reg:CC CC_REGNUM))
6809    (clobber (match_scratch:DI 1 "=r"))
6810    (use (reg:DI FP_REGNUM))]
6811   "TARGET_TLS_DESC && !TARGET_SVE"
6812   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6813   [(set_attr "type" "call")
6814    (set_attr "length" "16")])
6815
6816 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6817 ;; all vector registers, and clobber all predicate registers, on
6818 ;; top of the usual R0 and LR.
6819 (define_insn "tlsdesc_small_sve_<mode>"
6820   [(set (reg:PTR R0_REGNUM)
6821         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6822                     UNSPEC_TLSDESC))
6823    (clobber (reg:DI LR_REGNUM))
6824    (clobber (reg:CC CC_REGNUM))
6825    (clobber_high (reg:TI V0_REGNUM))
6826    (clobber_high (reg:TI V1_REGNUM))
6827    (clobber_high (reg:TI V2_REGNUM))
6828    (clobber_high (reg:TI V3_REGNUM))
6829    (clobber_high (reg:TI V4_REGNUM))
6830    (clobber_high (reg:TI V5_REGNUM))
6831    (clobber_high (reg:TI V6_REGNUM))
6832    (clobber_high (reg:TI V7_REGNUM))
6833    (clobber_high (reg:TI V8_REGNUM))
6834    (clobber_high (reg:TI V9_REGNUM))
6835    (clobber_high (reg:TI V10_REGNUM))
6836    (clobber_high (reg:TI V11_REGNUM))
6837    (clobber_high (reg:TI V12_REGNUM))
6838    (clobber_high (reg:TI V13_REGNUM))
6839    (clobber_high (reg:TI V14_REGNUM))
6840    (clobber_high (reg:TI V15_REGNUM))
6841    (clobber_high (reg:TI V16_REGNUM))
6842    (clobber_high (reg:TI V17_REGNUM))
6843    (clobber_high (reg:TI V18_REGNUM))
6844    (clobber_high (reg:TI V19_REGNUM))
6845    (clobber_high (reg:TI V20_REGNUM))
6846    (clobber_high (reg:TI V21_REGNUM))
6847    (clobber_high (reg:TI V22_REGNUM))
6848    (clobber_high (reg:TI V23_REGNUM))
6849    (clobber_high (reg:TI V24_REGNUM))
6850    (clobber_high (reg:TI V25_REGNUM))
6851    (clobber_high (reg:TI V26_REGNUM))
6852    (clobber_high (reg:TI V27_REGNUM))
6853    (clobber_high (reg:TI V28_REGNUM))
6854    (clobber_high (reg:TI V29_REGNUM))
6855    (clobber_high (reg:TI V30_REGNUM))
6856    (clobber_high (reg:TI V31_REGNUM))
6857    (clobber (reg:VNx2BI P0_REGNUM))
6858    (clobber (reg:VNx2BI P1_REGNUM))
6859    (clobber (reg:VNx2BI P2_REGNUM))
6860    (clobber (reg:VNx2BI P3_REGNUM))
6861    (clobber (reg:VNx2BI P4_REGNUM))
6862    (clobber (reg:VNx2BI P5_REGNUM))
6863    (clobber (reg:VNx2BI P6_REGNUM))
6864    (clobber (reg:VNx2BI P7_REGNUM))
6865    (clobber (reg:VNx2BI P8_REGNUM))
6866    (clobber (reg:VNx2BI P9_REGNUM))
6867    (clobber (reg:VNx2BI P10_REGNUM))
6868    (clobber (reg:VNx2BI P11_REGNUM))
6869    (clobber (reg:VNx2BI P12_REGNUM))
6870    (clobber (reg:VNx2BI P13_REGNUM))
6871    (clobber (reg:VNx2BI P14_REGNUM))
6872    (clobber (reg:VNx2BI P15_REGNUM))
6873    (clobber (match_scratch:DI 1 "=r"))
6874    (use (reg:DI FP_REGNUM))]
6875   "TARGET_TLS_DESC && TARGET_SVE"
6876   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6877   [(set_attr "type" "call")
6878    (set_attr "length" "16")])
6879
6880 (define_insn "stack_tie"
6881   [(set (mem:BLK (scratch))
6882         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6883                      (match_operand:DI 1 "register_operand" "rk")]
6884                     UNSPEC_PRLG_STK))]
6885   ""
6886   ""
6887   [(set_attr "length" "0")]
6888 )
6889
6890 ;; Pointer authentication patterns are always provided.  In architecture
6891 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6892 ;; This lets the user write portable software which authenticates pointers
6893 ;; when run on something which implements ARMv8.3-A, and which runs
6894 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6895 ;; implemented.
6896
6897 ;; Signing/Authenticating R30 using SP as the salt.
6898
6899 (define_insn "<pauth_mnem_prefix>sp"
6900   [(set (reg:DI R30_REGNUM)
6901         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6902   ""
6903   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6904 )
6905
6906 ;; Signing/Authenticating X17 using X16 as the salt.
6907
6908 (define_insn "<pauth_mnem_prefix>1716"
6909   [(set (reg:DI R17_REGNUM)
6910         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6911   ""
6912   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6913 )
6914
6915 ;; Stripping the signature in R30.
6916
6917 (define_insn "xpaclri"
6918   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6919   ""
6920   "hint\t7 // xpaclri"
6921 )
6922
6923 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6924 ;; all of memory.  This blocks insns from being moved across this point.
6925
6926 (define_insn "blockage"
6927   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6928   ""
6929   ""
6930   [(set_attr "length" "0")
6931    (set_attr "type" "block")]
6932 )
6933
6934 (define_insn "probe_stack_range"
6935   [(set (match_operand:DI 0 "register_operand" "=rk")
6936         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6937                              (match_operand:DI 2 "register_operand" "r")]
6938                               UNSPECV_PROBE_STACK_RANGE))]
6939   ""
6940 {
6941   return aarch64_output_probe_stack_range (operands[0], operands[2]);
6942 }
6943   [(set_attr "length" "32")]
6944 )
6945
6946 ;; This instruction is used to generate the stack clash stack adjustment and
6947 ;; probing loop.  We can't change the control flow during prologue and epilogue
6948 ;; code generation.  So we must emit a volatile unspec and expand it later on.
6949
6950 (define_insn "@probe_sve_stack_clash_<mode>"
6951   [(set (match_operand:P 0 "register_operand" "=rk")
6952         (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6953                             (match_operand:P 2 "register_operand" "r")
6954                             (match_operand:P 3 "const_int_operand" "n")
6955                             (match_operand:P 4 "aarch64_plus_immediate" "L")]
6956                              UNSPECV_PROBE_STACK_RANGE))]
6957   "TARGET_SVE"
6958 {
6959   return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6960                                                operands[3], operands[4]);
6961 }
6962   [(set_attr "length" "28")]
6963 )
6964
6965 ;; Named pattern for expanding thread pointer reference.
6966 (define_expand "get_thread_pointerdi"
6967   [(match_operand:DI 0 "register_operand")]
6968   ""
6969 {
6970   rtx tmp = aarch64_load_tp (operands[0]);
6971   if (tmp != operands[0])
6972     emit_move_insn (operands[0], tmp);
6973   DONE;
6974 })
6975
6976 ;; Named patterns for stack smashing protection.
6977 (define_expand "stack_protect_set"
6978   [(match_operand 0 "memory_operand")
6979    (match_operand 1 "memory_operand")]
6980   ""
6981 {
6982   machine_mode mode = GET_MODE (operands[0]);
6983   if (aarch64_stack_protector_guard != SSP_GLOBAL)
6984   {
6985     /* Generate access through the system register.  */
6986     rtx tmp_reg = gen_reg_rtx (mode);
6987     if (mode == DImode)
6988     {
6989         emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6990         emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6991                                GEN_INT (aarch64_stack_protector_guard_offset)));
6992     }
6993     else
6994     {
6995         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
6996         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
6997                                GEN_INT (aarch64_stack_protector_guard_offset)));
6998
6999     }
7000     operands[1] = gen_rtx_MEM (mode, tmp_reg);
7001   }
7002
7003   emit_insn ((mode == DImode
7004               ? gen_stack_protect_set_di
7005               : gen_stack_protect_set_si) (operands[0], operands[1]));
7006   DONE;
7007 })
7008
7009 (define_insn "reg_stack_protect_address_<mode>"
7010  [(set (match_operand:PTR 0 "register_operand" "=r")
7011        (unspec:PTR [(const_int 0)]
7012         UNSPEC_SSP_SYSREG))]
7013  "aarch64_stack_protector_guard != SSP_GLOBAL"
7014  {
7015    char buf[150];
7016    snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7017             aarch64_stack_protector_guard_reg_str);
7018    output_asm_insn (buf, operands);
7019    return "";
7020  }
7021  [(set_attr "type" "mrs")])
7022
7023 ;; DO NOT SPLIT THIS PATTERN.  It is important for security reasons that the
7024 ;; canary value does not live beyond the life of this sequence.
7025 (define_insn "stack_protect_set_<mode>"
7026   [(set (match_operand:PTR 0 "memory_operand" "=m")
7027         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7028          UNSPEC_SP_SET))
7029    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7030   ""
7031   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
7032   [(set_attr "length" "12")
7033    (set_attr "type" "multiple")])
7034
7035 (define_expand "stack_protect_test"
7036   [(match_operand 0 "memory_operand")
7037    (match_operand 1 "memory_operand")
7038    (match_operand 2)]
7039   ""
7040 {
7041   rtx result;
7042   machine_mode mode = GET_MODE (operands[0]);
7043
7044   result = gen_reg_rtx(mode);
7045   if (aarch64_stack_protector_guard != SSP_GLOBAL)
7046   {
7047     /* Generate access through the system register. The
7048        sequence we want here is the access
7049        of the stack offset to come with
7050        mrs scratch_reg, <system_register>
7051        add scratch_reg, scratch_reg, :lo12:offset. */
7052     rtx tmp_reg = gen_reg_rtx (mode);
7053     if (mode == DImode)
7054     {
7055        emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7056        emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7057                               GEN_INT (aarch64_stack_protector_guard_offset)));
7058     }
7059     else
7060     {
7061         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7062         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7063                                GEN_INT (aarch64_stack_protector_guard_offset)));
7064
7065     }
7066     operands[1] = gen_rtx_MEM (mode, tmp_reg);
7067   }
7068   emit_insn ((mode == DImode
7069                   ? gen_stack_protect_test_di
7070                   : gen_stack_protect_test_si) (result,
7071                                                 operands[0],
7072                                                 operands[1]));
7073
7074   if (mode == DImode)
7075     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7076                                     result, const0_rtx, operands[2]));
7077   else
7078     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7079                                     result, const0_rtx, operands[2]));
7080   DONE;
7081 })
7082
7083 (define_insn "stack_protect_test_<mode>"
7084   [(set (match_operand:PTR 0 "register_operand" "=r")
7085         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7086                      (match_operand:PTR 2 "memory_operand" "m")]
7087          UNSPEC_SP_TEST))
7088    (clobber (match_scratch:PTR 3 "=&r"))]
7089   ""
7090   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7091   [(set_attr "length" "12")
7092    (set_attr "type" "multiple")])
7093
7094 ;; Write Floating-point Control Register.
7095 (define_insn "set_fpcr"
7096   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7097   ""
7098   "msr\\tfpcr, %0"
7099   [(set_attr "type" "mrs")])
7100
7101 ;; Read Floating-point Control Register.
7102 (define_insn "get_fpcr"
7103   [(set (match_operand:SI 0 "register_operand" "=r")
7104         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7105   ""
7106   "mrs\\t%0, fpcr"
7107   [(set_attr "type" "mrs")])
7108
7109 ;; Write Floating-point Status Register.
7110 (define_insn "set_fpsr"
7111   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7112   ""
7113   "msr\\tfpsr, %0"
7114   [(set_attr "type" "mrs")])
7115
7116 ;; Read Floating-point Status Register.
7117 (define_insn "get_fpsr"
7118   [(set (match_operand:SI 0 "register_operand" "=r")
7119         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7120   ""
7121   "mrs\\t%0, fpsr"
7122   [(set_attr "type" "mrs")])
7123
7124
7125 ;; Define the subtract-one-and-jump insns so loop.c
7126 ;; knows what to generate.
7127 (define_expand "doloop_end"
7128   [(use (match_operand 0 "" ""))      ; loop pseudo
7129    (use (match_operand 1 "" ""))]     ; label
7130   "optimize > 0 && flag_modulo_sched"
7131 {
7132   rtx s0;
7133   rtx bcomp;
7134   rtx loc_ref;
7135   rtx cc_reg;
7136   rtx insn;
7137   rtx cmp;
7138
7139   /* Currently SMS relies on the do-loop pattern to recognize loops
7140      where (1) the control part consists of all insns defining and/or
7141      using a certain 'count' register and (2) the loop count can be
7142      adjusted by modifying this register prior to the loop.
7143      ??? The possible introduction of a new block to initialize the
7144      new IV can potentially affect branch optimizations.  */
7145
7146   if (GET_MODE (operands[0]) != DImode)
7147     FAIL;
7148
7149   s0 = operands [0];
7150   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7151
7152   cmp = XVECEXP (PATTERN (insn), 0, 0);
7153   cc_reg = SET_DEST (cmp);
7154   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7155   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7156   emit_jump_insn (gen_rtx_SET (pc_rtx,
7157                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7158                                                      loc_ref, pc_rtx)));
7159   DONE;
7160 })
7161
7162 ;; Track speculation through conditional branches.  We assume that
7163 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7164 (define_insn "speculation_tracker"
7165   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7166         (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7167          UNSPEC_SPECULATION_TRACKER))]
7168   ""
7169   {
7170     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7171     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7172     return "";
7173   }
7174   [(set_attr "type" "csel")]
7175 )
7176
7177 ;; BTI <target> instructions
7178 (define_insn "bti_noarg"
7179   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7180   ""
7181   "hint\t32 // bti"
7182   [(set_attr "type" "no_insn")]
7183 )
7184
7185 (define_insn "bti_c"
7186   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7187   ""
7188   "hint\t34 // bti c"
7189   [(set_attr "type" "no_insn")]
7190 )
7191
7192 (define_insn "bti_j"
7193   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7194   ""
7195   "hint\t36 // bti j"
7196   [(set_attr "type" "no_insn")]
7197 )
7198
7199 (define_insn "bti_jc"
7200   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7201   ""
7202   "hint\t38 // bti jc"
7203   [(set_attr "type" "no_insn")]
7204 )
7205
7206 ;; Hard speculation barrier.
7207 (define_insn "speculation_barrier"
7208   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7209   ""
7210   "isb\;dsb\\tsy"
7211   [(set_attr "length" "8")
7212    (set_attr "type" "block")
7213    (set_attr "speculation_barrier" "true")]
7214 )
7215
7216 ;; Support for __builtin_speculation_safe_value when we have speculation
7217 ;; tracking enabled.  Use the speculation tracker to decide whether to
7218 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7219 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7220   [(set (match_operand:ALLI_TI 0 "register_operand")
7221         (unspec_volatile:ALLI_TI
7222          [(match_operand:ALLI_TI 1 "register_operand")
7223           (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7224           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7225           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7226   ""
7227   "
7228   {
7229     if (operands[2] == const0_rtx)
7230       {
7231         rtx tracker;
7232         if (<MODE>mode == TImode)
7233           tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7234         else
7235           tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7236
7237         emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7238                                                  tracker));
7239         DONE;
7240       }
7241   }
7242   "
7243 )
7244
7245 ;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
7246 ;; encoding for CSDB, but will work in older versions of the assembler.
7247 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7248   [(set (match_operand:ALLI 0 "register_operand" "=r")
7249         (unspec_volatile:ALLI
7250          [(match_operand:ALLI 1 "register_operand" "r")
7251           (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7252           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7253           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7254   ""
7255   {
7256     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7257     output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7258                      operands);
7259     return "";
7260   }
7261   [(set_attr "length" "12")
7262    (set_attr "type" "block")
7263    (set_attr "speculation_barrier" "true")]
7264 )
7265
7266 ;; Pattern to match despeculate_copyti
7267 (define_insn "*despeculate_copyti_insn"
7268   [(set (match_operand:TI 0 "register_operand" "=r")
7269         (unspec_volatile:TI
7270          [(match_operand:TI 1 "register_operand" "r")
7271           (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7272           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7273           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7274   ""
7275   {
7276     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7277     output_asm_insn
7278       ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7279        operands);
7280     return "";
7281   }
7282   [(set_attr "length" "16")
7283    (set_attr "type" "block")
7284    (set_attr "speculation_barrier" "true")]
7285 )
7286
7287 (define_insn "despeculate_simple<ALLI:mode>"
7288   [(set (match_operand:ALLI 0 "register_operand" "=r")
7289         (unspec_volatile:ALLI
7290          [(match_operand:ALLI 1 "register_operand" "r")
7291           (use (match_operand:ALLI 2 "register_operand" ""))]
7292          UNSPECV_SPECULATION_BARRIER))]
7293   ""
7294   "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7295   [(set_attr "type" "block")
7296    (set_attr "length" "8")
7297    (set_attr "speculation_barrier" "true")]
7298 )
7299
7300 (define_insn "despeculate_simpleti"
7301   [(set (match_operand:TI 0 "register_operand" "=r")
7302         (unspec_volatile:TI
7303          [(match_operand:TI 1 "register_operand" "r")
7304           (use (match_operand:DI 2 "register_operand" ""))]
7305          UNSPECV_SPECULATION_BARRIER))]
7306   ""
7307   "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7308   [(set_attr "type" "block")
7309    (set_attr "length" "12")
7310    (set_attr "speculation_barrier" "true")]
7311 )
7312
7313 (define_insn "aarch64_<frintnzs_op><mode>"
7314   [(set (match_operand:VSFDF 0 "register_operand" "=w")
7315         (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7316                       FRINTNZX))]
7317   "TARGET_FRINT && TARGET_FLOAT
7318    && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7319   "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7320   [(set_attr "type" "f_rint<stype>")]
7321 )
7322
7323 ;; Transactional Memory Extension (TME) instructions.
7324
7325 (define_insn "tstart"
7326   [(set (match_operand:DI 0 "register_operand" "=r")
7327         (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7328    (clobber (mem:BLK (scratch)))]
7329   "TARGET_TME"
7330   "tstart\\t%0"
7331   [(set_attr "type" "tme")]
7332 )
7333
7334 (define_insn "ttest"
7335   [(set (match_operand:DI 0 "register_operand" "=r")
7336         (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7337    (clobber (mem:BLK (scratch)))]
7338   "TARGET_TME"
7339   "ttest\\t%0"
7340   [(set_attr "type" "tme")]
7341 )
7342
7343 (define_insn "tcommit"
7344   [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7345    (clobber (mem:BLK (scratch)))]
7346   "TARGET_TME"
7347   "tcommit"
7348   [(set_attr "type" "tme")]
7349 )
7350
7351 (define_insn "tcancel"
7352   [(unspec_volatile:BLK
7353      [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7354    (clobber (mem:BLK (scratch)))]
7355   "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7356   "tcancel\\t#%0"
7357   [(set_attr "type" "tme")]
7358 )
7359
7360 ;; AdvSIMD Stuff
7361 (include "aarch64-simd.md")
7362
7363 ;; Atomic Operations
7364 (include "atomics.md")
7365
7366 ;; ldp/stp peephole patterns
7367 (include "aarch64-ldpstp.md")
7368
7369 ;; SVE.
7370 (include "aarch64-sve.md")
7371
7372 ;; SVE2.
7373 (include "aarch64-sve2.md")