gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (R17_REGNUM         17)
  42     (R18_REGNUM         18)
  43     (R19_REGNUM         19)
  44     (R20_REGNUM         20)
  45     (R21_REGNUM         21)
  46     (R22_REGNUM         22)
  47     (R23_REGNUM         23)
  48     (R24_REGNUM         24)
  49     (R25_REGNUM         25)
  50     (R26_REGNUM         26)
  51     (R27_REGNUM         27)
  52     (R28_REGNUM         28)
  53     (R29_REGNUM         29)
  54     (R30_REGNUM         30)
  55     (SP_REGNUM          31)
  56     (V0_REGNUM          32)
  57     (V1_REGNUM          33)
  58     (V2_REGNUM          34)
  59     (V3_REGNUM          35)
  60     (V4_REGNUM          36)
  61     (V5_REGNUM          37)
  62     (V6_REGNUM          38)
  63     (V7_REGNUM          39)
  64     (V8_REGNUM          40)
  65     (V9_REGNUM          41)
  66     (V10_REGNUM         42)
  67     (V11_REGNUM         43)
  68     (V12_REGNUM         44)
  69     (V13_REGNUM         45)
  70     (V14_REGNUM         46)
  71     (V15_REGNUM         47)
  72     (V16_REGNUM         48)
  73     (V17_REGNUM         49)
  74     (V18_REGNUM         50)
  75     (V19_REGNUM         51)
  76     (V20_REGNUM         52)
  77     (V21_REGNUM         53)
  78     (V22_REGNUM         54)
  79     (V23_REGNUM         55)
  80     (V24_REGNUM         56)
  81     (V25_REGNUM         57)
  82     (V26_REGNUM         58)
  83     (V27_REGNUM         59)
  84     (V28_REGNUM         60)
  85     (V29_REGNUM         61)
  86     (V30_REGNUM         62)
  87     (V31_REGNUM         63)
  88     (LAST_SAVED_REGNUM  63)
  89     (SFP_REGNUM         64)
  90     (AP_REGNUM          65)
  91     (CC_REGNUM          66)
  92     ;; Defined only to make the DWARF description simpler.
  93     (VG_REGNUM          67)
  94     (P0_REGNUM          68)
  95     (P1_REGNUM          69)
  96     (P2_REGNUM          70)
  97     (P3_REGNUM          71)
  98     (P4_REGNUM          72)
  99     (P5_REGNUM          73)
 100     (P6_REGNUM          74)
 101     (P7_REGNUM          75)
 102     (P8_REGNUM          76)
 103     (P9_REGNUM          77)
 104     (P10_REGNUM         78)
 105     (P11_REGNUM         79)
 106     (P12_REGNUM         80)
 107     (P13_REGNUM         81)
 108     (P14_REGNUM         82)
 109     (P15_REGNUM         83)
 110     ;; Scratch register used by stack clash protection to calculate
 111     ;; SVE CFA offsets during probing.
 112     (STACK_CLASH_SVE_CFA_REGNUM 11)
 113     ;; Scratch registers for prologue/epilogue use.
 114     (EP0_REGNUM         12)
 115     (EP1_REGNUM         13)
 116     ;; A couple of call-clobbered registers that we need to reserve when
 117     ;; tracking speculation this is not ABI, so is subject to change.
 118     (SPECULATION_SCRATCH_REGNUM 14)
 119     (SPECULATION_TRACKER_REGNUM 15)
 120     ;; Scratch registers used in frame layout.
 121     (IP0_REGNUM         16)
 122     (IP1_REGNUM         17)
 123     (FP_REGNUM          29)
 124     (LR_REGNUM          30)
 125   ]
 126 )
 127
 128 (define_c_enum "unspec" [
 129     UNSPEC_AUTIA1716
 130     UNSPEC_AUTIB1716
 131     UNSPEC_AUTIASP
 132     UNSPEC_AUTIBSP
 133     UNSPEC_CASESI
 134     UNSPEC_CRC32B
 135     UNSPEC_CRC32CB
 136     UNSPEC_CRC32CH
 137     UNSPEC_CRC32CW
 138     UNSPEC_CRC32CX
 139     UNSPEC_CRC32H
 140     UNSPEC_CRC32W
 141     UNSPEC_CRC32X
 142     UNSPEC_FCVTZS
 143     UNSPEC_FCVTZU
 144     UNSPEC_URECPE
 145     UNSPEC_FRECPE
 146     UNSPEC_FRECPS
 147     UNSPEC_FRECPX
 148     UNSPEC_FRINTA
 149     UNSPEC_FRINTI
 150     UNSPEC_FRINTM
 151     UNSPEC_FRINTN
 152     UNSPEC_FRINTP
 153     UNSPEC_FRINTX
 154     UNSPEC_FRINTZ
 155     UNSPEC_GOTSMALLPIC
 156     UNSPEC_GOTSMALLPIC28K
 157     UNSPEC_GOTSMALLTLS
 158     UNSPEC_GOTTINYPIC
 159     UNSPEC_GOTTINYTLS
 160     UNSPEC_LD1
 161     UNSPEC_LD2
 162     UNSPEC_LD2_DREG
 163     UNSPEC_LD2_DUP
 164     UNSPEC_LD3
 165     UNSPEC_LD3_DREG
 166     UNSPEC_LD3_DUP
 167     UNSPEC_LD4
 168     UNSPEC_LD4_DREG
 169     UNSPEC_LD4_DUP
 170     UNSPEC_LD2_LANE
 171     UNSPEC_LD3_LANE
 172     UNSPEC_LD4_LANE
 173     UNSPEC_MB
 174     UNSPEC_NOP
 175     UNSPEC_PACIA1716
 176     UNSPEC_PACIB1716
 177     UNSPEC_PACIASP
 178     UNSPEC_PACIBSP
 179     UNSPEC_PRLG_STK
 180     UNSPEC_REV
 181     UNSPEC_RBIT
 182     UNSPEC_SABAL
 183     UNSPEC_SABDL2
 184     UNSPEC_SADALP
 185     UNSPEC_SCVTF
 186     UNSPEC_SISD_NEG
 187     UNSPEC_SISD_SSHL
 188     UNSPEC_SISD_USHL
 189     UNSPEC_SSHL_2S
 190     UNSPEC_ST1
 191     UNSPEC_ST2
 192     UNSPEC_ST3
 193     UNSPEC_ST4
 194     UNSPEC_ST2_LANE
 195     UNSPEC_ST3_LANE
 196     UNSPEC_ST4_LANE
 197     UNSPEC_TLS
 198     UNSPEC_TLSDESC
 199     UNSPEC_TLSLE12
 200     UNSPEC_TLSLE24
 201     UNSPEC_TLSLE32
 202     UNSPEC_TLSLE48
 203     UNSPEC_UABAL
 204     UNSPEC_UABDL2
 205     UNSPEC_UADALP
 206     UNSPEC_UCVTF
 207     UNSPEC_USHL_2S
 208     UNSPEC_VSTRUCTDUMMY
 209     UNSPEC_SSP_SYSREG
 210     UNSPEC_SP_SET
 211     UNSPEC_SP_TEST
 212     UNSPEC_RSQRT
 213     UNSPEC_RSQRTE
 214     UNSPEC_RSQRTS
 215     UNSPEC_NZCV
 216     UNSPEC_XPACLRI
 217     UNSPEC_LD1_SVE
 218     UNSPEC_ST1_SVE
 219     UNSPEC_LD1RQ
 220     UNSPEC_LD1_GATHER
 221     UNSPEC_ST1_SCATTER
 222     UNSPEC_MERGE_PTRUE
 223     UNSPEC_PTEST_PTRUE
 224     UNSPEC_UNPACKSHI
 225     UNSPEC_UNPACKUHI
 226     UNSPEC_UNPACKSLO
 227     UNSPEC_UNPACKULO
 228     UNSPEC_PACK
 229     UNSPEC_FLOAT_CONVERT
 230     UNSPEC_WHILE_LO
 231     UNSPEC_LDN
 232     UNSPEC_STN
 233     UNSPEC_INSR
 234     UNSPEC_CLASTB
 235     UNSPEC_FADDA
 236     UNSPEC_REV_SUBREG
 237     UNSPEC_REINTERPRET
 238     UNSPEC_SPECULATION_TRACKER
 239     UNSPEC_COPYSIGN
 240     UNSPEC_TTEST                ; Represent transaction test.
 241 ])
 242
 243 (define_c_enum "unspecv" [
 244     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 245     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 246     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 247     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 248     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 249     UNSPECV_BLOCKAGE            ; Represent a blockage
 250     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 251     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 252     UNSPECV_BTI_NOARG           ; Represent BTI.
 253     UNSPECV_BTI_C               ; Represent BTI c.
 254     UNSPECV_BTI_J               ; Represent BTI j.
 255     UNSPECV_BTI_JC              ; Represent BTI jc.
 256     UNSPECV_TSTART              ; Represent transaction start.
 257     UNSPECV_TCOMMIT             ; Represent transaction commit.
 258     UNSPECV_TCANCEL             ; Represent transaction cancel.
 259   ]
 260 )
 261
 262 ;; If further include files are added the defintion of MD_INCLUDES
 263 ;; must be updated.
 264
 265 (include "constraints.md")
 266 (include "predicates.md")
 267 (include "iterators.md")
 268
 269 ;; -------------------------------------------------------------------
 270 ;; Instruction types and attributes
 271 ;; -------------------------------------------------------------------
 272
 273 ; The "type" attribute is included here from AArch32 backend to be able
 274 ; to share pipeline descriptions.
 275 (include "../arm/types.md")
 276
 277 ;; It is important to set the fp or simd attributes to yes when a pattern
 278 ;; alternative uses the FP or SIMD register files, usually signified by use of
 279 ;; the 'w' constraint.  This will ensure that the alternative will be
 280 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 281 ;; architecture extensions.  If all the alternatives in a pattern use the
 282 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 283 ;; or TARGET_SIMD.
 284
 285 ;; Attributes of the architecture required to support the instruction (or
 286 ;; alternative). This attribute is used to compute attribute "enabled", use type
 287 ;; "any" to enable an alternative in all cases.
 288
 289 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
 290
 291 (define_enum_attr "arch" "arches" (const_string "any"))
 292
 293 ;; [For compatibility with Arm in pipeline models]
 294 ;; Attribute that specifies whether or not the instruction touches fp
 295 ;; registers.
 296 ;; Note that this attribute is not used anywhere in either the arm or aarch64
 297 ;; backends except in the scheduling description for xgene1.  In that
 298 ;; scheduling description this attribute is used to subclass the load_4 and
 299 ;; load_8 types.
 300 (define_attr "fp" "no,yes"
 301   (if_then_else
 302     (eq_attr "arch" "fp")
 303     (const_string "yes")
 304     (const_string "no")))
 305
 306 (define_attr "arch_enabled" "no,yes"
 307   (if_then_else
 308     (ior
 309         (eq_attr "arch" "any")
 310
 311         (and (eq_attr "arch" "rcpc8_4")
 312              (match_test "AARCH64_ISA_RCPC8_4"))
 313
 314         (and (eq_attr "arch" "fp")
 315              (match_test "TARGET_FLOAT"))
 316
 317         (and (eq_attr "arch" "simd")
 318              (match_test "TARGET_SIMD"))
 319
 320         (and (eq_attr "arch" "fp16")
 321              (match_test "TARGET_FP_F16INST"))
 322
 323         (and (eq_attr "arch" "sve")
 324              (match_test "TARGET_SVE")))
 325     (const_string "yes")
 326     (const_string "no")))
 327
 328 ;; Attribute that controls whether an alternative is enabled or not.
 329 ;; Currently it is only used to disable alternatives which touch fp or simd
 330 ;; registers when -mgeneral-regs-only is specified or to require a special
 331 ;; architecture support.
 332 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
 333
 334 ;; Attribute that specifies whether we are dealing with a branch to a
 335 ;; label that is far away, i.e. further away than the maximum/minimum
 336 ;; representable in a signed 21-bits number.
 337 ;; 0 :=: no
 338 ;; 1 :=: yes
 339 (define_attr "far_branch" "" (const_int 0))
 340
 341 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 342 (define_attr "movprfx" "no,yes" (const_string "no"))
 343
 344 (define_attr "length" ""
 345   (cond [(eq_attr "movprfx" "yes")
 346            (const_int 8)
 347         ] (const_int 4)))
 348
 349 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 350 ;; no predicated insns.
 351 (define_attr "predicated" "yes,no" (const_string "no"))
 352
 353 ;; Set to true on an insn that requires the speculation tracking state to be
 354 ;; in the tracking register before the insn issues.  Otherwise the compiler
 355 ;; may chose to hold the tracking state encoded in SP.
 356 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 357
 358 ;; -------------------------------------------------------------------
 359 ;; Pipeline descriptions and scheduling
 360 ;; -------------------------------------------------------------------
 361
 362 ;; Processor types.
 363 (include "aarch64-tune.md")
 364
 365 ;; Scheduling
 366 (include "../arm/cortex-a53.md")
 367 (include "../arm/cortex-a57.md")
 368 (include "../arm/exynos-m1.md")
 369 (include "falkor.md")
 370 (include "saphira.md")
 371 (include "thunderx.md")
 372 (include "../arm/xgene1.md")
 373 (include "thunderx2t99.md")
 374 (include "tsv110.md")
 375
 376 ;; -------------------------------------------------------------------
 377 ;; Jumps and other miscellaneous insns
 378 ;; -------------------------------------------------------------------
 379
 380 (define_insn "indirect_jump"
 381   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 382   ""
 383   "br\\t%0"
 384   [(set_attr "type" "branch")]
 385 )
 386
 387 (define_insn "jump"
 388   [(set (pc) (label_ref (match_operand 0 "" "")))]
 389   ""
 390   "b\\t%l0"
 391   [(set_attr "type" "branch")]
 392 )
 393
 394 (define_expand "cbranch<mode>4"
 395   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 396                             [(match_operand:GPI 1 "register_operand")
 397                              (match_operand:GPI 2 "aarch64_plus_operand")])
 398                            (label_ref (match_operand 3 "" ""))
 399                            (pc)))]
 400   ""
 401   "
 402   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 403                                          operands[2]);
 404   operands[2] = const0_rtx;
 405   "
 406 )
 407
 408 (define_expand "cbranch<mode>4"
 409   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 410                             [(match_operand:GPF 1 "register_operand")
 411                              (match_operand:GPF 2 "aarch64_fp_compare_operand")])
 412                            (label_ref (match_operand 3 "" ""))
 413                            (pc)))]
 414   ""
 415   "
 416   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 417                                          operands[2]);
 418   operands[2] = const0_rtx;
 419   "
 420 )
 421
 422 (define_expand "cbranchcc4"
 423   [(set (pc) (if_then_else
 424               (match_operator 0 "aarch64_comparison_operator"
 425                [(match_operand 1 "cc_register")
 426                 (match_operand 2 "const0_operand")])
 427               (label_ref (match_operand 3 "" ""))
 428               (pc)))]
 429   ""
 430   "")
 431
 432 (define_insn "ccmp<mode>"
 433   [(set (match_operand:CC 1 "cc_register" "")
 434         (if_then_else:CC
 435           (match_operator 4 "aarch64_comparison_operator"
 436            [(match_operand 0 "cc_register" "")
 437             (const_int 0)])
 438           (compare:CC
 439             (match_operand:GPI 2 "register_operand" "r,r,r")
 440             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 441           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 442   ""
 443   "@
 444    ccmp\\t%<w>2, %<w>3, %k5, %m4
 445    ccmp\\t%<w>2, %3, %k5, %m4
 446    ccmn\\t%<w>2, #%n3, %k5, %m4"
 447   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 448 )
 449
 450 (define_insn "fccmp<mode>"
 451   [(set (match_operand:CCFP 1 "cc_register" "")
 452         (if_then_else:CCFP
 453           (match_operator 4 "aarch64_comparison_operator"
 454            [(match_operand 0 "cc_register" "")
 455             (const_int 0)])
 456           (compare:CCFP
 457             (match_operand:GPF 2 "register_operand" "w")
 458             (match_operand:GPF 3 "register_operand" "w"))
 459           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 460   "TARGET_FLOAT"
 461   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 462   [(set_attr "type" "fccmp<s>")]
 463 )
 464
 465 (define_insn "fccmpe<mode>"
 466   [(set (match_operand:CCFPE 1 "cc_register" "")
 467          (if_then_else:CCFPE
 468           (match_operator 4 "aarch64_comparison_operator"
 469            [(match_operand 0 "cc_register" "")
 470           (const_int 0)])
 471            (compare:CCFPE
 472             (match_operand:GPF 2 "register_operand" "w")
 473             (match_operand:GPF 3 "register_operand" "w"))
 474           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 475   "TARGET_FLOAT"
 476   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 477   [(set_attr "type" "fccmp<s>")]
 478 )
 479
 480 ;; Expansion of signed mod by a power of 2 using CSNEG.
 481 ;; For x0 % n where n is a power of 2 produce:
 482 ;; negs   x1, x0
 483 ;; and    x0, x0, #(n - 1)
 484 ;; and    x1, x1, #(n - 1)
 485 ;; csneg  x0, x0, x1, mi
 486
 487 (define_expand "mod<mode>3"
 488   [(match_operand:GPI 0 "register_operand")
 489    (match_operand:GPI 1 "register_operand")
 490    (match_operand:GPI 2 "const_int_operand")]
 491   ""
 492   {
 493     HOST_WIDE_INT val = INTVAL (operands[2]);
 494
 495     if (val <= 0
 496        || exact_log2 (val) <= 0
 497        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 498       FAIL;
 499
 500     rtx mask = GEN_INT (val - 1);
 501
 502     /* In the special case of x0 % 2 we can do the even shorter:
 503         cmp    x0, xzr
 504         and    x0, x0, 1
 505         cneg   x0, x0, lt.  */
 506     if (val == 2)
 507       {
 508         rtx masked = gen_reg_rtx (<MODE>mode);
 509         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 510         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 511         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 512         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 513         DONE;
 514       }
 515
 516     rtx neg_op = gen_reg_rtx (<MODE>mode);
 517     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 518
 519     /* Extract the condition register and mode.  */
 520     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 521     rtx cc_reg = SET_DEST (cmp);
 522     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 523
 524     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 525     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 526
 527     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 528     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 529
 530     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 531                                        masked_neg, masked_pos));
 532     DONE;
 533   }
 534 )
 535
 536 (define_insn "condjump"
 537   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 538                             [(match_operand 1 "cc_register" "") (const_int 0)])
 539                            (label_ref (match_operand 2 "" ""))
 540                            (pc)))]
 541   ""
 542   {
 543     /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
 544        but the "." is required for SVE conditions.  */
 545     bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
 546     if (get_attr_length (insn) == 8)
 547       return aarch64_gen_far_branch (operands, 2, "Lbcond",
 548                                      use_dot_p ? "b.%M0\\t" : "b%M0\\t");
 549     else
 550       return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
 551   }
 552   [(set_attr "type" "branch")
 553    (set (attr "length")
 554         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 555                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 556                       (const_int 4)
 557                       (const_int 8)))
 558    (set (attr "far_branch")
 559         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 560                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 561                       (const_int 0)
 562                       (const_int 1)))]
 563 )
 564
 565 ;; For a 24-bit immediate CST we can optimize the compare for equality
 566 ;; and branch sequence from:
 567 ;;      mov     x0, #imm1
 568 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 569 ;;      cmp     x1, x0
 570 ;;      b<ne,eq> .Label
 571 ;; into the shorter:
 572 ;;      sub     x0, x1, #(CST & 0xfff000)
 573 ;;      subs    x0, x0, #(CST & 0x000fff)
 574 ;;      b<ne,eq> .Label
 575 (define_insn_and_split "*compare_condjump<GPI:mode>"
 576   [(set (pc) (if_then_else (EQL
 577                               (match_operand:GPI 0 "register_operand" "r")
 578                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 579                            (label_ref:P (match_operand 2 "" ""))
 580                            (pc)))]
 581   "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
 582    && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
 583    && !reload_completed"
 584   "#"
 585   "&& true"
 586   [(const_int 0)]
 587   {
 588     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 589     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 590     rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
 591     emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 592     emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 593     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 594     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
 595                                   cc_reg, const0_rtx);
 596     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 597     DONE;
 598   }
 599 )
 600
 601 (define_expand "casesi"
 602   [(match_operand:SI 0 "register_operand")      ; Index
 603    (match_operand:SI 1 "const_int_operand")     ; Lower bound
 604    (match_operand:SI 2 "const_int_operand")     ; Total range
 605    (match_operand:DI 3 "" "")                   ; Table label
 606    (match_operand:DI 4 "" "")]                  ; Out of range label
 607   ""
 608   {
 609     if (operands[1] != const0_rtx)
 610       {
 611         rtx reg = gen_reg_rtx (SImode);
 612
 613         /* Canonical RTL says that if you have:
 614
 615            (minus (X) (CONST))
 616
 617            then this should be emitted as:
 618
 619            (plus (X) (-CONST))
 620
 621            The use of trunc_int_for_mode ensures that the resulting
 622            constant can be represented in SImode, this is important
 623            for the corner case where operand[1] is INT_MIN.  */
 624
 625         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 626
 627         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 628               (operands[1], SImode))
 629           operands[1] = force_reg (SImode, operands[1]);
 630         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 631         operands[0] = reg;
 632       }
 633
 634     if (!aarch64_plus_operand (operands[2], SImode))
 635       operands[2] = force_reg (SImode, operands[2]);
 636     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 637                                                  const0_rtx),
 638                                     operands[0], operands[2], operands[4]));
 639
 640     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 641     operands[2]
 642       = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
 643                         UNSPEC_CASESI);
 644     operands[2] = gen_rtx_MEM (DImode, operands[2]);
 645     MEM_READONLY_P (operands[2]) = 1;
 646     MEM_NOTRAP_P (operands[2]) = 1;
 647     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
 648     DONE;
 649   }
 650 )
 651
 652 (define_expand "casesi_dispatch"
 653   [(parallel
 654     [(set (pc) (match_operand:DI 0 ""))
 655      (clobber (reg:CC CC_REGNUM))
 656      (clobber (match_scratch:DI 2))
 657      (clobber (match_scratch:DI 3))
 658      (use (label_ref:DI (match_operand 1 "")))])]
 659   "")
 660
 661 (define_insn "*casesi_dispatch"
 662   [(parallel
 663     [(set (pc)
 664           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 665                            (match_operand:SI 1 "register_operand" "r")]
 666                         UNSPEC_CASESI)))
 667      (clobber (reg:CC CC_REGNUM))
 668      (clobber (match_scratch:DI 3 "=r"))
 669      (clobber (match_scratch:DI 4 "=r"))
 670      (use (label_ref:DI (match_operand 2 "" "")))])]
 671   ""
 672   "*
 673   return aarch64_output_casesi (operands);
 674   "
 675   [(set_attr "length" "16")
 676    (set_attr "type" "branch")]
 677 )
 678
 679 (define_insn "nop"
 680   [(unspec[(const_int 0)] UNSPEC_NOP)]
 681   ""
 682   "nop"
 683   [(set_attr "type" "no_insn")]
 684 )
 685
 686 (define_insn "prefetch"
 687   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 688             (match_operand:QI 1 "const_int_operand" "")
 689             (match_operand:QI 2 "const_int_operand" ""))]
 690   ""
 691   {
 692     const char * pftype[2][4] =
 693     {
 694       {"prfm\\tPLDL1STRM, %0",
 695        "prfm\\tPLDL3KEEP, %0",
 696        "prfm\\tPLDL2KEEP, %0",
 697        "prfm\\tPLDL1KEEP, %0"},
 698       {"prfm\\tPSTL1STRM, %0",
 699        "prfm\\tPSTL3KEEP, %0",
 700        "prfm\\tPSTL2KEEP, %0",
 701        "prfm\\tPSTL1KEEP, %0"},
 702     };
 703
 704     int locality = INTVAL (operands[2]);
 705
 706     gcc_assert (IN_RANGE (locality, 0, 3));
 707
 708     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 709        the address into a DImode MEM so that aarch64_print_operand knows
 710        how to print it.  */
 711     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 712     return pftype[INTVAL(operands[1])][locality];
 713   }
 714   [(set_attr "type" "load_4")]
 715 )
 716
 717 (define_insn "trap"
 718   [(trap_if (const_int 1) (const_int 8))]
 719   ""
 720   "brk #1000"
 721   [(set_attr "type" "trap")])
 722
 723 (define_expand "prologue"
 724   [(clobber (const_int 0))]
 725   ""
 726   "
 727   aarch64_expand_prologue ();
 728   DONE;
 729   "
 730 )
 731
 732 (define_expand "epilogue"
 733   [(clobber (const_int 0))]
 734   ""
 735   "
 736   aarch64_expand_epilogue (false);
 737   DONE;
 738   "
 739 )
 740
 741 (define_expand "sibcall_epilogue"
 742   [(clobber (const_int 0))]
 743   ""
 744   "
 745   aarch64_expand_epilogue (true);
 746   DONE;
 747   "
 748 )
 749
 750 (define_insn "*do_return"
 751   [(return)]
 752   ""
 753   {
 754     if (aarch64_return_address_signing_enabled ()
 755         && TARGET_ARMV8_3
 756         && !crtl->calls_eh_return)
 757       {
 758         if (aarch64_ra_sign_key == AARCH64_KEY_B)
 759           return "retab";
 760         else
 761           return "retaa";
 762       }
 763     return "ret";
 764   }
 765   [(set_attr "type" "branch")]
 766 )
 767
 768 (define_expand "return"
 769   [(simple_return)]
 770   "aarch64_use_return_insn_p ()"
 771   ""
 772 )
 773
 774 (define_insn "simple_return"
 775   [(simple_return)]
 776   "aarch64_use_simple_return_insn_p ()"
 777   "ret"
 778   [(set_attr "type" "branch")]
 779 )
 780
 781 (define_insn "*cb<optab><mode>1"
 782   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 783                                 (const_int 0))
 784                            (label_ref (match_operand 1 "" ""))
 785                            (pc)))]
 786   "!aarch64_track_speculation"
 787   {
 788     if (get_attr_length (insn) == 8)
 789       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 790     else
 791       return "<cbz>\\t%<w>0, %l1";
 792   }
 793   [(set_attr "type" "branch")
 794    (set (attr "length")
 795         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 796                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 797                       (const_int 4)
 798                       (const_int 8)))
 799    (set (attr "far_branch")
 800         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 801                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 802                       (const_int 0)
 803                       (const_int 1)))]
 804 )
 805
 806 (define_insn "*tb<optab><mode>1"
 807   [(set (pc) (if_then_else
 808               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 809                                     (const_int 1)
 810                                     (match_operand 1
 811                                       "aarch64_simd_shift_imm_<mode>" "n"))
 812                    (const_int 0))
 813              (label_ref (match_operand 2 "" ""))
 814              (pc)))
 815    (clobber (reg:CC CC_REGNUM))]
 816   "!aarch64_track_speculation"
 817   {
 818     if (get_attr_length (insn) == 8)
 819       {
 820         if (get_attr_far_branch (insn) == 1)
 821           return aarch64_gen_far_branch (operands, 2, "Ltb",
 822                                          "<inv_tb>\\t%<w>0, %1, ");
 823         else
 824           {
 825             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 826             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 827           }
 828       }
 829     else
 830       return "<tbz>\t%<w>0, %1, %l2";
 831   }
 832   [(set_attr "type" "branch")
 833    (set (attr "length")
 834         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 835                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 836                       (const_int 4)
 837                       (const_int 8)))
 838    (set (attr "far_branch")
 839         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 840                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 841                       (const_int 0)
 842                       (const_int 1)))]
 843
 844 )
 845
 846 (define_insn "*cb<optab><mode>1"
 847   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 848                                  (const_int 0))
 849                            (label_ref (match_operand 1 "" ""))
 850                            (pc)))
 851    (clobber (reg:CC CC_REGNUM))]
 852   "!aarch64_track_speculation"
 853   {
 854     if (get_attr_length (insn) == 8)
 855       {
 856         if (get_attr_far_branch (insn) == 1)
 857           return aarch64_gen_far_branch (operands, 1, "Ltb",
 858                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 859         else
 860           {
 861             char buf[64];
 862             uint64_t val = ((uint64_t) 1)
 863                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 864             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 865             output_asm_insn (buf, operands);
 866             return "<bcond>\t%l1";
 867           }
 868       }
 869     else
 870       return "<tbz>\t%<w>0, <sizem1>, %l1";
 871   }
 872   [(set_attr "type" "branch")
 873    (set (attr "length")
 874         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 875                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 876                       (const_int 4)
 877                       (const_int 8)))
 878    (set (attr "far_branch")
 879         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 880                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 881                       (const_int 0)
 882                       (const_int 1)))]
 883 )
 884
 885 ;; -------------------------------------------------------------------
 886 ;; Subroutine calls and sibcalls
 887 ;; -------------------------------------------------------------------
 888
 889 (define_expand "call"
 890   [(parallel [(call (match_operand 0 "memory_operand")
 891                     (match_operand 1 "general_operand"))
 892               (use (match_operand 2 "" ""))
 893               (clobber (reg:DI LR_REGNUM))])]
 894   ""
 895   "
 896   {
 897     aarch64_expand_call (NULL_RTX, operands[0], false);
 898     DONE;
 899   }"
 900 )
 901
 902 (define_insn "*call_insn"
 903   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 904          (match_operand 1 "" ""))
 905    (clobber (reg:DI LR_REGNUM))]
 906   ""
 907   "@
 908   blr\\t%0
 909   bl\\t%c0"
 910   [(set_attr "type" "call, call")]
 911 )
 912
 913 (define_expand "call_value"
 914   [(parallel [(set (match_operand 0 "" "")
 915                    (call (match_operand 1 "memory_operand")
 916                          (match_operand 2 "general_operand")))
 917               (use (match_operand 3 "" ""))
 918               (clobber (reg:DI LR_REGNUM))])]
 919   ""
 920   "
 921   {
 922     aarch64_expand_call (operands[0], operands[1], false);
 923     DONE;
 924   }"
 925 )
 926
 927 (define_insn "*call_value_insn"
 928   [(set (match_operand 0 "" "")
 929         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 930                       (match_operand 2 "" "")))
 931    (clobber (reg:DI LR_REGNUM))]
 932   ""
 933   "@
 934   blr\\t%1
 935   bl\\t%c1"
 936   [(set_attr "type" "call, call")]
 937 )
 938
 939 (define_expand "sibcall"
 940   [(parallel [(call (match_operand 0 "memory_operand")
 941                     (match_operand 1 "general_operand"))
 942               (return)
 943               (use (match_operand 2 "" ""))])]
 944   ""
 945   {
 946     aarch64_expand_call (NULL_RTX, operands[0], true);
 947     DONE;
 948   }
 949 )
 950
 951 (define_expand "sibcall_value"
 952   [(parallel [(set (match_operand 0 "" "")
 953                    (call (match_operand 1 "memory_operand")
 954                          (match_operand 2 "general_operand")))
 955               (return)
 956               (use (match_operand 3 "" ""))])]
 957   ""
 958   {
 959     aarch64_expand_call (operands[0], operands[1], true);
 960     DONE;
 961   }
 962 )
 963
 964 (define_insn "*sibcall_insn"
 965   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 966          (match_operand 1 "" ""))
 967    (return)]
 968   "SIBLING_CALL_P (insn)"
 969   "@
 970    br\\t%0
 971    b\\t%c0"
 972   [(set_attr "type" "branch, branch")]
 973 )
 974
 975 (define_insn "*sibcall_value_insn"
 976   [(set (match_operand 0 "" "")
 977         (call (mem:DI
 978                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 979               (match_operand 2 "" "")))
 980    (return)]
 981   "SIBLING_CALL_P (insn)"
 982   "@
 983    br\\t%1
 984    b\\t%c1"
 985   [(set_attr "type" "branch, branch")]
 986 )
 987
 988 ;; Call subroutine returning any type.
 989
 990 (define_expand "untyped_call"
 991   [(parallel [(call (match_operand 0 "")
 992                     (const_int 0))
 993               (match_operand 1 "")
 994               (match_operand 2 "")])]
 995   ""
 996 {
 997   int i;
 998
 999   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
1000
1001   for (i = 0; i < XVECLEN (operands[2], 0); i++)
1002     {
1003       rtx set = XVECEXP (operands[2], 0, i);
1004       emit_move_insn (SET_DEST (set), SET_SRC (set));
1005     }
1006
1007   /* The optimizer does not know that the call sets the function value
1008      registers we stored in the result block.  We avoid problems by
1009      claiming that all hard registers are used and clobbered at this
1010      point.  */
1011   emit_insn (gen_blockage ());
1012   DONE;
1013 })
1014
1015 ;; -------------------------------------------------------------------
1016 ;; Moves
1017 ;; -------------------------------------------------------------------
1018
1019 (define_expand "mov<mode>"
1020   [(set (match_operand:SHORT 0 "nonimmediate_operand")
1021         (match_operand:SHORT 1 "general_operand"))]
1022   ""
1023   "
1024     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1025       operands[1] = force_reg (<MODE>mode, operands[1]);
1026
1027     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1028       {
1029         aarch64_expand_mov_immediate (operands[0], operands[1]);
1030         DONE;
1031       }
1032   "
1033 )
1034
1035 (define_insn "*mov<mode>_aarch64"
1036   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
1037         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1038   "(register_operand (operands[0], <MODE>mode)
1039     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1040 {
1041    switch (which_alternative)
1042      {
1043      case 0:
1044        return "mov\t%w0, %w1";
1045      case 1:
1046        return "mov\t%w0, %1";
1047      case 2:
1048        return aarch64_output_scalar_simd_mov_immediate (operands[1],
1049                                                         <MODE>mode);
1050      case 3:
1051        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1052      case 4:
1053        return "ldr<size>\t%w0, %1";
1054      case 5:
1055        return "ldr\t%<size>0, %1";
1056      case 6:
1057        return "str<size>\t%w1, %0";
1058      case 7:
1059        return "str\t%<size>1, %0";
1060      case 8:
1061        return "umov\t%w0, %1.<v>[0]";
1062      case 9:
1063        return "dup\t%0.<Vallxd>, %w1";
1064      case 10:
1065        return "dup\t%<Vetype>0, %1.<v>[0]";
1066      default:
1067        gcc_unreachable ();
1068      }
1069 }
1070   ;; The "mov_imm" type for CNT is just a placeholder.
1071   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1072                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1073    (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1074 )
1075
1076 (define_expand "mov<mode>"
1077   [(set (match_operand:GPI 0 "nonimmediate_operand")
1078         (match_operand:GPI 1 "general_operand"))]
1079   ""
1080   "
1081     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1082         && <MODE>mode == DImode
1083         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1084       DONE;
1085
1086     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1087       operands[1] = force_reg (<MODE>mode, operands[1]);
1088
1089     /* FIXME: RR we still need to fix up what we are doing with
1090        symbol_refs and other types of constants.  */
1091     if (CONSTANT_P (operands[1])
1092         && !CONST_INT_P (operands[1]))
1093      {
1094        aarch64_expand_mov_immediate (operands[0], operands[1]);
1095        DONE;
1096      }
1097   "
1098 )
1099
1100 (define_insn_and_split "*movsi_aarch64"
1101   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1102         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1103   "(register_operand (operands[0], SImode)
1104     || aarch64_reg_or_zero (operands[1], SImode))"
1105   "@
1106    mov\\t%w0, %w1
1107    mov\\t%w0, %w1
1108    mov\\t%w0, %w1
1109    mov\\t%w0, %1
1110    #
1111    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1112    ldr\\t%w0, %1
1113    ldr\\t%s0, %1
1114    str\\t%w1, %0
1115    str\\t%s1, %0
1116    adr\\t%x0, %c1
1117    adrp\\t%x0, %A1
1118    fmov\\t%s0, %w1
1119    fmov\\t%w0, %s1
1120    fmov\\t%s0, %s1
1121    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1122   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1123     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1124    [(const_int 0)]
1125    "{
1126        aarch64_expand_mov_immediate (operands[0], operands[1]);
1127        DONE;
1128     }"
1129   ;; The "mov_imm" type for CNT is just a placeholder.
1130   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1131                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1132    (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1133 )
1134
1135 (define_insn_and_split "*movdi_aarch64"
1136   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1137         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1138   "(register_operand (operands[0], DImode)
1139     || aarch64_reg_or_zero (operands[1], DImode))"
1140   "@
1141    mov\\t%x0, %x1
1142    mov\\t%0, %x1
1143    mov\\t%x0, %1
1144    mov\\t%x0, %1
1145    mov\\t%w0, %1
1146    #
1147    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1148    ldr\\t%x0, %1
1149    ldr\\t%d0, %1
1150    str\\t%x1, %0
1151    str\\t%d1, %0
1152    adr\\t%x0, %c1
1153    adrp\\t%x0, %A1
1154    fmov\\t%d0, %x1
1155    fmov\\t%x0, %d1
1156    fmov\\t%d0, %d1
1157    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1158    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1159     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1160    [(const_int 0)]
1161    "{
1162        aarch64_expand_mov_immediate (operands[0], operands[1]);
1163        DONE;
1164     }"
1165   ;; The "mov_imm" type for CNTD is just a placeholder.
1166   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1167                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1168                      neon_move")
1169    (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1170 )
1171
1172 (define_insn "insv_imm<mode>"
1173   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1174                           (const_int 16)
1175                           (match_operand:GPI 1 "const_int_operand" "n"))
1176         (match_operand:GPI 2 "const_int_operand" "n"))]
1177   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1178    && UINTVAL (operands[1]) % 16 == 0"
1179   "movk\\t%<w>0, %X2, lsl %1"
1180   [(set_attr "type" "mov_imm")]
1181 )
1182
1183 (define_expand "movti"
1184   [(set (match_operand:TI 0 "nonimmediate_operand")
1185         (match_operand:TI 1 "general_operand"))]
1186   ""
1187   "
1188     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1189       operands[1] = force_reg (TImode, operands[1]);
1190
1191     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1192       {
1193         emit_move_insn (gen_lowpart (DImode, operands[0]),
1194                         gen_lowpart (DImode, operands[1]));
1195         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1196         DONE;
1197       }
1198   "
1199 )
1200
1201 (define_insn "*movti_aarch64"
1202   [(set (match_operand:TI 0
1203          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1204         (match_operand:TI 1
1205          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1206   "(register_operand (operands[0], TImode)
1207     || aarch64_reg_or_zero (operands[1], TImode))"
1208   "@
1209    #
1210    #
1211    #
1212    mov\\t%0.16b, %1.16b
1213    ldp\\t%0, %H0, %1
1214    stp\\t%1, %H1, %0
1215    stp\\txzr, xzr, %0
1216    ldr\\t%q0, %1
1217    str\\t%q1, %0"
1218   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1219                              load_16,store_16,store_16,\
1220                              load_16,store_16")
1221    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1222    (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1223 )
1224
1225 ;; Split a TImode register-register or register-immediate move into
1226 ;; its component DImode pieces, taking care to handle overlapping
1227 ;; source and dest registers.
1228 (define_split
1229    [(set (match_operand:TI 0 "register_operand" "")
1230          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1231   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1232   [(const_int 0)]
1233 {
1234   aarch64_split_128bit_move (operands[0], operands[1]);
1235   DONE;
1236 })
1237
1238 (define_expand "mov<mode>"
1239   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1240         (match_operand:GPF_TF_F16 1 "general_operand"))]
1241   ""
1242   {
1243     if (!TARGET_FLOAT)
1244       {
1245         aarch64_err_no_fpadvsimd (<MODE>mode);
1246         FAIL;
1247       }
1248
1249     if (GET_CODE (operands[0]) == MEM
1250         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1251               && aarch64_float_const_zero_rtx_p (operands[1])))
1252       operands[1] = force_reg (<MODE>mode, operands[1]);
1253   }
1254 )
1255
1256 (define_insn "*movhf_aarch64"
1257   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1258         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1259   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1260     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1261   "@
1262    movi\\t%0.4h, #0
1263    fmov\\t%h0, %w1
1264    dup\\t%w0.4h, %w1
1265    umov\\t%w0, %1.h[0]
1266    mov\\t%0.h[0], %1.h[0]
1267    fmov\\t%h0, %1
1268    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1269    ldr\\t%h0, %1
1270    str\\t%h1, %0
1271    ldrh\\t%w0, %1
1272    strh\\t%w1, %0
1273    mov\\t%w0, %w1"
1274   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1275                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1276    (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1277 )
1278
1279 (define_insn "*movsf_aarch64"
1280   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1281         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1282   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1283     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1284   "@
1285    movi\\t%0.2s, #0
1286    fmov\\t%s0, %w1
1287    fmov\\t%w0, %s1
1288    fmov\\t%s0, %s1
1289    fmov\\t%s0, %1
1290    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1291    ldr\\t%s0, %1
1292    str\\t%s1, %0
1293    ldr\\t%w0, %1
1294    str\\t%w1, %0
1295    mov\\t%w0, %w1
1296    mov\\t%w0, %1"
1297   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1298                      f_loads,f_stores,load_4,store_4,mov_reg,\
1299                      fconsts")
1300    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1301 )
1302
1303 (define_insn "*movdf_aarch64"
1304   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1305         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1306   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1307     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1308   "@
1309    movi\\t%d0, #0
1310    fmov\\t%d0, %x1
1311    fmov\\t%x0, %d1
1312    fmov\\t%d0, %d1
1313    fmov\\t%d0, %1
1314    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1315    ldr\\t%d0, %1
1316    str\\t%d1, %0
1317    ldr\\t%x0, %1
1318    str\\t%x1, %0
1319    mov\\t%x0, %x1
1320    mov\\t%x0, %1"
1321   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1322                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1323                      fconstd")
1324    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1325 )
1326
1327 (define_split
1328   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1329         (match_operand:GPF_HF 1 "general_operand"))]
1330   "can_create_pseudo_p ()
1331    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1332    && !aarch64_float_const_representable_p (operands[1])
1333    &&  aarch64_float_const_rtx_p (operands[1])"
1334   [(const_int 0)]
1335   {
1336     unsigned HOST_WIDE_INT ival;
1337     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1338       FAIL;
1339
1340     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1341     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1342     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1343     DONE;
1344   }
1345 )
1346
1347 (define_insn "*movtf_aarch64"
1348   [(set (match_operand:TF 0
1349          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1350         (match_operand:TF 1
1351          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1352   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1353     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1354   "@
1355    mov\\t%0.16b, %1.16b
1356    #
1357    #
1358    #
1359    movi\\t%0.2d, #0
1360    fmov\\t%s0, wzr
1361    ldr\\t%q0, %1
1362    str\\t%q1, %0
1363    ldp\\t%0, %H0, %1
1364    stp\\t%1, %H1, %0
1365    stp\\txzr, xzr, %0"
1366   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1367                      f_loadd,f_stored,load_16,store_16,store_16")
1368    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1369    (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1370 )
1371
1372 (define_split
1373    [(set (match_operand:TF 0 "register_operand" "")
1374          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1375   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1376   [(const_int 0)]
1377   {
1378     aarch64_split_128bit_move (operands[0], operands[1]);
1379     DONE;
1380   }
1381 )
1382
1383 ;; 0 is dst
1384 ;; 1 is src
1385 ;; 2 is size of copy in bytes
1386 ;; 3 is alignment
1387
1388 (define_expand "cpymemdi"
1389   [(match_operand:BLK 0 "memory_operand")
1390    (match_operand:BLK 1 "memory_operand")
1391    (match_operand:DI 2 "immediate_operand")
1392    (match_operand:DI 3 "immediate_operand")]
1393    "!STRICT_ALIGNMENT"
1394 {
1395   if (aarch64_expand_cpymem (operands))
1396     DONE;
1397   FAIL;
1398 }
1399 )
1400
1401 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1402 ;; fairly lax checking on the second memory operation.
1403 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1404   [(set (match_operand:SX 0 "register_operand" "=r,w")
1405         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1406    (set (match_operand:SX2 2 "register_operand" "=r,w")
1407         (match_operand:SX2 3 "memory_operand" "m,m"))]
1408    "rtx_equal_p (XEXP (operands[3], 0),
1409                  plus_constant (Pmode,
1410                                 XEXP (operands[1], 0),
1411                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1412   "@
1413    ldp\\t%w0, %w2, %1
1414    ldp\\t%s0, %s2, %1"
1415   [(set_attr "type" "load_8,neon_load1_2reg")
1416    (set_attr "arch" "*,fp")]
1417 )
1418
1419 ;; Storing different modes that can still be merged
1420 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1421   [(set (match_operand:DX 0 "register_operand" "=r,w")
1422         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1423    (set (match_operand:DX2 2 "register_operand" "=r,w")
1424         (match_operand:DX2 3 "memory_operand" "m,m"))]
1425    "rtx_equal_p (XEXP (operands[3], 0),
1426                  plus_constant (Pmode,
1427                                 XEXP (operands[1], 0),
1428                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1429   "@
1430    ldp\\t%x0, %x2, %1
1431    ldp\\t%d0, %d2, %1"
1432   [(set_attr "type" "load_16,neon_load1_2reg")
1433    (set_attr "arch" "*,fp")]
1434 )
1435
1436 (define_insn "load_pair_dw_tftf"
1437   [(set (match_operand:TF 0 "register_operand" "=w")
1438         (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1439    (set (match_operand:TF 2 "register_operand" "=w")
1440         (match_operand:TF 3 "memory_operand" "m"))]
1441    "TARGET_SIMD
1442     && rtx_equal_p (XEXP (operands[3], 0),
1443                     plus_constant (Pmode,
1444                                    XEXP (operands[1], 0),
1445                                    GET_MODE_SIZE (TFmode)))"
1446   "ldp\\t%q0, %q2, %1"
1447   [(set_attr "type" "neon_ldp_q")
1448    (set_attr "fp" "yes")]
1449 )
1450
1451 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1452 ;; fairly lax checking on the second memory operation.
1453 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1454   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1455         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1456    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1457         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1458    "rtx_equal_p (XEXP (operands[2], 0),
1459                  plus_constant (Pmode,
1460                                 XEXP (operands[0], 0),
1461                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1462   "@
1463    stp\\t%w1, %w3, %0
1464    stp\\t%s1, %s3, %0"
1465   [(set_attr "type" "store_8,neon_store1_2reg")
1466    (set_attr "arch" "*,fp")]
1467 )
1468
1469 ;; Storing different modes that can still be merged
1470 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1471   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1472         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1473    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1474         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1475    "rtx_equal_p (XEXP (operands[2], 0),
1476                  plus_constant (Pmode,
1477                                 XEXP (operands[0], 0),
1478                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1479   "@
1480    stp\\t%x1, %x3, %0
1481    stp\\t%d1, %d3, %0"
1482   [(set_attr "type" "store_16,neon_store1_2reg")
1483    (set_attr "arch" "*,fp")]
1484 )
1485
1486 (define_insn "store_pair_dw_tftf"
1487   [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1488         (match_operand:TF 1 "register_operand" "w"))
1489    (set (match_operand:TF 2 "memory_operand" "=m")
1490         (match_operand:TF 3 "register_operand" "w"))]
1491    "TARGET_SIMD &&
1492     rtx_equal_p (XEXP (operands[2], 0),
1493                  plus_constant (Pmode,
1494                                 XEXP (operands[0], 0),
1495                                 GET_MODE_SIZE (TFmode)))"
1496   "stp\\t%q1, %q3, %0"
1497   [(set_attr "type" "neon_stp_q")
1498    (set_attr "fp" "yes")]
1499 )
1500
1501 ;; Load pair with post-index writeback.  This is primarily used in function
1502 ;; epilogues.
1503 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1504   [(parallel
1505     [(set (match_operand:P 0 "register_operand" "=k")
1506           (plus:P (match_operand:P 1 "register_operand" "0")
1507                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1508      (set (match_operand:GPI 2 "register_operand" "=r")
1509           (mem:GPI (match_dup 1)))
1510      (set (match_operand:GPI 3 "register_operand" "=r")
1511           (mem:GPI (plus:P (match_dup 1)
1512                    (match_operand:P 5 "const_int_operand" "n"))))])]
1513   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1514   "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1515   [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1516 )
1517
1518 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1519   [(parallel
1520     [(set (match_operand:P 0 "register_operand" "=k")
1521           (plus:P (match_operand:P 1 "register_operand" "0")
1522                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1523      (set (match_operand:GPF 2 "register_operand" "=w")
1524           (mem:GPF (match_dup 1)))
1525      (set (match_operand:GPF 3 "register_operand" "=w")
1526           (mem:GPF (plus:P (match_dup 1)
1527                    (match_operand:P 5 "const_int_operand" "n"))))])]
1528   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1529   "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1530   [(set_attr "type" "neon_load1_2reg")]
1531 )
1532
1533 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1534   [(parallel
1535     [(set (match_operand:P 0 "register_operand" "=k")
1536           (plus:P (match_operand:P 1 "register_operand" "0")
1537                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1538      (set (match_operand:TX 2 "register_operand" "=w")
1539           (mem:TX (match_dup 1)))
1540      (set (match_operand:TX 3 "register_operand" "=w")
1541           (mem:TX (plus:P (match_dup 1)
1542                           (match_operand:P 5 "const_int_operand" "n"))))])]
1543   "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1544   "ldp\\t%q2, %q3, [%1], %4"
1545   [(set_attr "type" "neon_ldp_q")]
1546 )
1547
1548 ;; Store pair with pre-index writeback.  This is primarily used in function
1549 ;; prologues.
1550 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1551   [(parallel
1552     [(set (match_operand:P 0 "register_operand" "=&k")
1553           (plus:P (match_operand:P 1 "register_operand" "0")
1554                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1555      (set (mem:GPI (plus:P (match_dup 0)
1556                    (match_dup 4)))
1557           (match_operand:GPI 2 "register_operand" "r"))
1558      (set (mem:GPI (plus:P (match_dup 0)
1559                    (match_operand:P 5 "const_int_operand" "n")))
1560           (match_operand:GPI 3 "register_operand" "r"))])]
1561   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1562   "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1563   [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1564 )
1565
1566 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1567   [(parallel
1568     [(set (match_operand:P 0 "register_operand" "=&k")
1569           (plus:P (match_operand:P 1 "register_operand" "0")
1570                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1571      (set (mem:GPF (plus:P (match_dup 0)
1572                    (match_dup 4)))
1573           (match_operand:GPF 2 "register_operand" "w"))
1574      (set (mem:GPF (plus:P (match_dup 0)
1575                    (match_operand:P 5 "const_int_operand" "n")))
1576           (match_operand:GPF 3 "register_operand" "w"))])]
1577   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1578   "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1579   [(set_attr "type" "neon_store1_2reg<q>")]
1580 )
1581
1582 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1583   [(parallel
1584     [(set (match_operand:P 0 "register_operand" "=&k")
1585           (plus:P (match_operand:P 1 "register_operand" "0")
1586                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1587      (set (mem:TX (plus:P (match_dup 0)
1588                           (match_dup 4)))
1589           (match_operand:TX 2 "register_operand" "w"))
1590      (set (mem:TX (plus:P (match_dup 0)
1591                           (match_operand:P 5 "const_int_operand" "n")))
1592           (match_operand:TX 3 "register_operand" "w"))])]
1593   "TARGET_SIMD
1594    && INTVAL (operands[5])
1595       == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1596   "stp\\t%q2, %q3, [%0, %4]!"
1597   [(set_attr "type" "neon_stp_q")]
1598 )
1599
1600 ;; -------------------------------------------------------------------
1601 ;; Sign/Zero extension
1602 ;; -------------------------------------------------------------------
1603
1604 (define_expand "<optab>sidi2"
1605   [(set (match_operand:DI 0 "register_operand")
1606         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1607   ""
1608 )
1609
1610 (define_insn "*extendsidi2_aarch64"
1611   [(set (match_operand:DI 0 "register_operand" "=r,r")
1612         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1613   ""
1614   "@
1615    sxtw\t%0, %w1
1616    ldrsw\t%0, %1"
1617   [(set_attr "type" "extend,load_4")]
1618 )
1619
1620 (define_insn "*load_pair_extendsidi2_aarch64"
1621   [(set (match_operand:DI 0 "register_operand" "=r")
1622         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1623    (set (match_operand:DI 2 "register_operand" "=r")
1624         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1625   "rtx_equal_p (XEXP (operands[3], 0),
1626                 plus_constant (Pmode,
1627                                XEXP (operands[1], 0),
1628                                GET_MODE_SIZE (SImode)))"
1629   "ldpsw\\t%0, %2, %1"
1630   [(set_attr "type" "load_8")]
1631 )
1632
1633 (define_insn "*zero_extendsidi2_aarch64"
1634   [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1635         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1636   ""
1637   "@
1638    uxtw\t%0, %w1
1639    ldr\t%w0, %1
1640    fmov\t%s0, %w1
1641    ldr\t%s0, %1
1642    fmov\t%w0, %s1
1643    fmov\t%s0, %s1"
1644   [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1645    (set_attr "arch" "*,*,fp,fp,fp,fp")]
1646 )
1647
1648 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1649   [(set (match_operand:DI 0 "register_operand" "=r,w")
1650         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1651    (set (match_operand:DI 2 "register_operand" "=r,w")
1652         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1653   "rtx_equal_p (XEXP (operands[3], 0),
1654                 plus_constant (Pmode,
1655                                XEXP (operands[1], 0),
1656                                GET_MODE_SIZE (SImode)))"
1657   "@
1658    ldp\t%w0, %w2, %1
1659    ldp\t%s0, %s2, %1"
1660   [(set_attr "type" "load_8,neon_load1_2reg")
1661    (set_attr "arch" "*,fp")]
1662 )
1663
1664 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1665   [(set (match_operand:GPI 0 "register_operand")
1666         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1667   ""
1668 )
1669
1670 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1671   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1672         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1673   ""
1674   "@
1675    sxt<SHORT:size>\t%<GPI:w>0, %w1
1676    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1677   [(set_attr "type" "extend,load_4")]
1678 )
1679
1680 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1681   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1682         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1683   ""
1684   "@
1685    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1686    ldr<SHORT:size>\t%w0, %1
1687    ldr\t%<SHORT:size>0, %1"
1688   [(set_attr "type" "logic_imm,load_4,f_loads")
1689    (set_attr "arch" "*,*,fp")]
1690 )
1691
1692 (define_expand "<optab>qihi2"
1693   [(set (match_operand:HI 0 "register_operand")
1694         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1695   ""
1696 )
1697
1698 (define_insn "*extendqihi2_aarch64"
1699   [(set (match_operand:HI 0 "register_operand" "=r,r")
1700         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1701   ""
1702   "@
1703    sxtb\t%w0, %w1
1704    ldrsb\t%w0, %1"
1705   [(set_attr "type" "extend,load_4")]
1706 )
1707
1708 (define_insn "*zero_extendqihi2_aarch64"
1709   [(set (match_operand:HI 0 "register_operand" "=r,r")
1710         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1711   ""
1712   "@
1713    and\t%w0, %w1, 255
1714    ldrb\t%w0, %1"
1715   [(set_attr "type" "logic_imm,load_4")]
1716 )
1717
1718 ;; -------------------------------------------------------------------
1719 ;; Simple arithmetic
1720 ;; -------------------------------------------------------------------
1721
1722 (define_expand "add<mode>3"
1723   [(set
1724     (match_operand:GPI 0 "register_operand")
1725     (plus:GPI (match_operand:GPI 1 "register_operand")
1726               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1727   ""
1728 {
1729   /* If operands[1] is a subreg extract the inner RTX.  */
1730   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1731
1732   /* If the constant is too large for a single instruction and isn't frame
1733      based, split off the immediate so it is available for CSE.  */
1734   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1735       && can_create_pseudo_p ()
1736       && (!REG_P (op1)
1737          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1738     operands[2] = force_reg (<MODE>mode, operands[2]);
1739   /* Expand polynomial additions now if the destination is the stack
1740      pointer, since we don't want to use that as a temporary.  */
1741   else if (operands[0] == stack_pointer_rtx
1742            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1743     {
1744       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1745                                 operands[2], NULL_RTX, NULL_RTX);
1746       DONE;
1747     }
1748 })
1749
1750 (define_insn "*add<mode>3_aarch64"
1751   [(set
1752     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1753     (plus:GPI
1754      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1755      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1756   ""
1757   "@
1758   add\\t%<w>0, %<w>1, %2
1759   add\\t%<w>0, %<w>1, %<w>2
1760   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1761   sub\\t%<w>0, %<w>1, #%n2
1762   #
1763   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1764   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1765   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1766    (set_attr "arch" "*,*,simd,*,*,*")]
1767 )
1768
1769 ;; zero_extend version of above
1770 (define_insn "*addsi3_aarch64_uxtw"
1771   [(set
1772     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1773     (zero_extend:DI
1774      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1775               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1776   ""
1777   "@
1778   add\\t%w0, %w1, %2
1779   add\\t%w0, %w1, %w2
1780   sub\\t%w0, %w1, #%n2
1781   #"
1782   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1783 )
1784
1785 ;; If there's a free register, and we can load the constant with a
1786 ;; single instruction, do so.  This has a chance to improve scheduling.
1787 (define_peephole2
1788   [(match_scratch:GPI 3 "r")
1789    (set (match_operand:GPI 0 "register_operand")
1790         (plus:GPI
1791           (match_operand:GPI 1 "register_operand")
1792           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1793   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1794   [(set (match_dup 3) (match_dup 2))
1795    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1796 )
1797
1798 (define_peephole2
1799   [(match_scratch:SI 3 "r")
1800    (set (match_operand:DI 0 "register_operand")
1801         (zero_extend:DI
1802           (plus:SI
1803             (match_operand:SI 1 "register_operand")
1804             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1805   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1806   [(set (match_dup 3) (match_dup 2))
1807    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1808 )
1809
1810 ;; After peephole2 has had a chance to run, split any remaining long
1811 ;; additions into two add immediates.
1812 (define_split
1813   [(set (match_operand:GPI 0 "register_operand")
1814         (plus:GPI
1815           (match_operand:GPI 1 "register_operand")
1816           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1817   "epilogue_completed"
1818   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1819    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1820   {
1821     HOST_WIDE_INT i = INTVAL (operands[2]);
1822     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1823     operands[3] = GEN_INT (i - s);
1824     operands[4] = GEN_INT (s);
1825   }
1826 )
1827
1828 ;; Match addition of polynomial offsets that require one temporary, for which
1829 ;; we can use the early-clobbered destination register.  This is a separate
1830 ;; pattern so that the early clobber doesn't affect register allocation
1831 ;; for other forms of addition.  However, we still need to provide an
1832 ;; all-register alternative, in case the offset goes out of range after
1833 ;; elimination.  For completeness we might as well provide all GPR-based
1834 ;; alternatives from the main pattern.
1835 ;;
1836 ;; We don't have a pattern for additions requiring two temporaries since at
1837 ;; present LRA doesn't allow new scratches to be added during elimination.
1838 ;; Such offsets should be rare anyway.
1839 ;;
1840 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1841 ;; here would go away.  We could just handle all polynomial constants in
1842 ;; this pattern.
1843 (define_insn_and_split "*add<mode>3_poly_1"
1844   [(set
1845     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1846     (plus:GPI
1847      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1848      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1849   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1850   "@
1851   add\\t%<w>0, %<w>1, %2
1852   add\\t%<w>0, %<w>1, %<w>2
1853   sub\\t%<w>0, %<w>1, #%n2
1854   #
1855   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1856   #"
1857   "&& epilogue_completed
1858    && !reg_overlap_mentioned_p (operands[0], operands[1])
1859    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1860   [(const_int 0)]
1861   {
1862     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1863                               operands[2], operands[0], NULL_RTX);
1864     DONE;
1865   }
1866   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1867   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1868 )
1869
1870 (define_split
1871   [(set (match_operand:DI 0 "register_operand")
1872         (zero_extend:DI
1873           (plus:SI
1874             (match_operand:SI 1 "register_operand")
1875             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1876   "epilogue_completed"
1877   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1878    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1879   {
1880     HOST_WIDE_INT i = INTVAL (operands[2]);
1881     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1882     operands[3] = GEN_INT (i - s);
1883     operands[4] = GEN_INT (s);
1884     operands[5] = gen_lowpart (SImode, operands[0]);
1885   }
1886 )
1887
1888 (define_expand "addv<mode>4"
1889   [(match_operand:GPI 0 "register_operand")
1890    (match_operand:GPI 1 "register_operand")
1891    (match_operand:GPI 2 "aarch64_plus_operand")
1892    (label_ref (match_operand 3 "" ""))]
1893   ""
1894 {
1895   if (CONST_INT_P (operands[2]))
1896     emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1897                                             operands[2]));
1898   else
1899     emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1900   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1901
1902   DONE;
1903 })
1904
1905 (define_expand "uaddv<mode>4"
1906   [(match_operand:GPI 0 "register_operand")
1907    (match_operand:GPI 1 "register_operand")
1908    (match_operand:GPI 2 "register_operand")
1909    (label_ref (match_operand 3 "" ""))]
1910   ""
1911 {
1912   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1913   aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1914
1915   DONE;
1916 })
1917
1918 (define_expand "addti3"
1919   [(set (match_operand:TI 0 "register_operand")
1920         (plus:TI (match_operand:TI 1 "register_operand")
1921                  (match_operand:TI 2 "aarch64_reg_or_imm")))]
1922   ""
1923 {
1924   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1925
1926   aarch64_addti_scratch_regs (operands[1], operands[2],
1927                               &low_dest, &op1_low, &op2_low,
1928                               &high_dest, &op1_high, &op2_high);
1929
1930   if (op2_low == const0_rtx)
1931     {
1932       low_dest = op1_low;
1933       if (!aarch64_pluslong_operand (op2_high, DImode))
1934         op2_high = force_reg (DImode, op2_high);
1935       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1936     }
1937   else
1938     {
1939       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1940                                       force_reg (DImode, op2_low)));
1941       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1942                                      force_reg (DImode, op2_high)));
1943     }
1944
1945   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1946   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1947
1948   DONE;
1949 })
1950
1951 (define_expand "addvti4"
1952   [(match_operand:TI 0 "register_operand")
1953    (match_operand:TI 1 "register_operand")
1954    (match_operand:TI 2 "aarch64_reg_or_imm")
1955    (label_ref (match_operand 3 "" ""))]
1956   ""
1957 {
1958   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1959
1960   aarch64_addti_scratch_regs (operands[1], operands[2],
1961                               &low_dest, &op1_low, &op2_low,
1962                               &high_dest, &op1_high, &op2_high);
1963
1964   if (op2_low == const0_rtx)
1965     {
1966       low_dest = op1_low;
1967       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1968                                       force_reg (DImode, op2_high)));
1969     }
1970   else
1971     {
1972       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1973                                       force_reg (DImode, op2_low)));
1974       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
1975                                       force_reg (DImode, op2_high)));
1976     }
1977
1978   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1979   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1980
1981   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1982   DONE;
1983 })
1984
1985 (define_expand "uaddvti4"
1986   [(match_operand:TI 0 "register_operand")
1987    (match_operand:TI 1 "register_operand")
1988    (match_operand:TI 2 "aarch64_reg_or_imm")
1989    (label_ref (match_operand 3 "" ""))]
1990   ""
1991 {
1992   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1993
1994   aarch64_addti_scratch_regs (operands[1], operands[2],
1995                               &low_dest, &op1_low, &op2_low,
1996                               &high_dest, &op1_high, &op2_high);
1997
1998   if (op2_low == const0_rtx)
1999     {
2000       low_dest = op1_low;
2001       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2002                                       force_reg (DImode, op2_high)));
2003     }
2004   else
2005     {
2006       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2007                                       force_reg (DImode, op2_low)));
2008       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2009                                       force_reg (DImode, op2_high)));
2010     }
2011
2012   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2013   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2014
2015   aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2016   DONE;
2017  })
2018
2019 (define_insn "add<mode>3_compare0"
2020   [(set (reg:CC_NZ CC_REGNUM)
2021         (compare:CC_NZ
2022          (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2023                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2024          (const_int 0)))
2025    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2026         (plus:GPI (match_dup 1) (match_dup 2)))]
2027   ""
2028   "@
2029   adds\\t%<w>0, %<w>1, %<w>2
2030   adds\\t%<w>0, %<w>1, %2
2031   subs\\t%<w>0, %<w>1, #%n2"
2032   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2033 )
2034
2035 ;; zero_extend version of above
2036 (define_insn "*addsi3_compare0_uxtw"
2037   [(set (reg:CC_NZ CC_REGNUM)
2038         (compare:CC_NZ
2039          (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2040                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2041          (const_int 0)))
2042    (set (match_operand:DI 0 "register_operand" "=r,r,r")
2043         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2044   ""
2045   "@
2046   adds\\t%w0, %w1, %w2
2047   adds\\t%w0, %w1, %2
2048   subs\\t%w0, %w1, #%n2"
2049   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2050 )
2051
2052 (define_insn "*add<mode>3_compareC_cconly"
2053   [(set (reg:CC_C CC_REGNUM)
2054         (compare:CC_C
2055           (plus:GPI
2056             (match_operand:GPI 0 "register_operand" "r,r,r")
2057             (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2058           (match_dup 0)))]
2059   ""
2060   "@
2061   cmn\\t%<w>0, %<w>1
2062   cmn\\t%<w>0, %1
2063   cmp\\t%<w>0, #%n1"
2064   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2065 )
2066
2067 (define_insn "add<mode>3_compareC"
2068   [(set (reg:CC_C CC_REGNUM)
2069         (compare:CC_C
2070           (plus:GPI
2071             (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2072             (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2073           (match_dup 1)))
2074    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2075         (plus:GPI (match_dup 1) (match_dup 2)))]
2076   ""
2077   "@
2078   adds\\t%<w>0, %<w>1, %<w>2
2079   adds\\t%<w>0, %<w>1, %2
2080   subs\\t%<w>0, %<w>1, #%n2"
2081   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2082 )
2083
2084 (define_insn "*add<mode>3_compareV_cconly_imm"
2085   [(set (reg:CC_V CC_REGNUM)
2086         (compare:CC_V
2087           (plus:<DWI>
2088             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2089             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2090           (sign_extend:<DWI>
2091            (plus:GPI
2092             (match_dup 0)
2093             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2094   "INTVAL (operands[1]) == INTVAL (operands[2])"
2095   "@
2096   cmn\\t%<w>0, %<w>1
2097   cmp\\t%<w>0, #%n1"
2098   [(set_attr "type" "alus_imm")]
2099 )
2100
2101 (define_insn "*add<mode>3_compareV_cconly"
2102   [(set (reg:CC_V CC_REGNUM)
2103         (compare:CC_V
2104           (plus:<DWI>
2105             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2106             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2107           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2108   ""
2109   "cmn\\t%<w>0, %<w>1"
2110   [(set_attr "type" "alus_sreg")]
2111 )
2112
2113 (define_insn "add<mode>3_compareV_imm"
2114   [(set (reg:CC_V CC_REGNUM)
2115         (compare:CC_V
2116           (plus:<DWI>
2117             (sign_extend:<DWI>
2118               (match_operand:GPI 1 "register_operand" "rk,rk"))
2119             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2120           (sign_extend:<DWI>
2121             (plus:GPI (match_dup 1) (match_dup 2)))))
2122    (set (match_operand:GPI 0 "register_operand" "=r,r")
2123         (plus:GPI (match_dup 1) (match_dup 2)))]
2124    ""
2125    "@
2126    adds\\t%<w>0, %<w>1, %<w>2
2127    subs\\t%<w>0, %<w>1, #%n2"
2128   [(set_attr "type" "alus_imm,alus_imm")]
2129 )
2130
2131 (define_insn "add<mode>3_compareV"
2132   [(set (reg:CC_V CC_REGNUM)
2133         (compare:CC_V
2134           (plus:<DWI>
2135             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2136             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2137           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2138    (set (match_operand:GPI 0 "register_operand" "=r")
2139         (plus:GPI (match_dup 1) (match_dup 2)))]
2140   ""
2141   "adds\\t%<w>0, %<w>1, %<w>2"
2142   [(set_attr "type" "alus_sreg")]
2143 )
2144
2145 (define_insn "*adds_shift_imm_<mode>"
2146   [(set (reg:CC_NZ CC_REGNUM)
2147         (compare:CC_NZ
2148          (plus:GPI (ASHIFT:GPI
2149                     (match_operand:GPI 1 "register_operand" "r")
2150                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2151                    (match_operand:GPI 3 "register_operand" "r"))
2152          (const_int 0)))
2153    (set (match_operand:GPI 0 "register_operand" "=r")
2154         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2155                   (match_dup 3)))]
2156   ""
2157   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2158   [(set_attr "type" "alus_shift_imm")]
2159 )
2160
2161 (define_insn "*subs_shift_imm_<mode>"
2162   [(set (reg:CC_NZ CC_REGNUM)
2163         (compare:CC_NZ
2164          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2165                     (ASHIFT:GPI
2166                      (match_operand:GPI 2 "register_operand" "r")
2167                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2168          (const_int 0)))
2169    (set (match_operand:GPI 0 "register_operand" "=r")
2170         (minus:GPI (match_dup 1)
2171                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2172   ""
2173   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2174   [(set_attr "type" "alus_shift_imm")]
2175 )
2176
2177 (define_insn "*adds_mul_imm_<mode>"
2178   [(set (reg:CC_NZ CC_REGNUM)
2179         (compare:CC_NZ
2180          (plus:GPI (mult:GPI
2181                     (match_operand:GPI 1 "register_operand" "r")
2182                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2183                    (match_operand:GPI 3 "register_operand" "r"))
2184          (const_int 0)))
2185    (set (match_operand:GPI 0 "register_operand" "=r")
2186         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2187                   (match_dup 3)))]
2188   ""
2189   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2190   [(set_attr "type" "alus_shift_imm")]
2191 )
2192
2193 (define_insn "*subs_mul_imm_<mode>"
2194   [(set (reg:CC_NZ CC_REGNUM)
2195         (compare:CC_NZ
2196          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2197                     (mult:GPI
2198                      (match_operand:GPI 2 "register_operand" "r")
2199                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2200          (const_int 0)))
2201    (set (match_operand:GPI 0 "register_operand" "=r")
2202         (minus:GPI (match_dup 1)
2203                    (mult:GPI (match_dup 2) (match_dup 3))))]
2204   ""
2205   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2206   [(set_attr "type" "alus_shift_imm")]
2207 )
2208
2209 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2210   [(set (reg:CC_NZ CC_REGNUM)
2211         (compare:CC_NZ
2212          (plus:GPI
2213           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2214           (match_operand:GPI 2 "register_operand" "rk"))
2215         (const_int 0)))
2216    (set (match_operand:GPI 0 "register_operand" "=r")
2217         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2218   ""
2219   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2220   [(set_attr "type" "alus_ext")]
2221 )
2222
2223 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2224   [(set (reg:CC_NZ CC_REGNUM)
2225         (compare:CC_NZ
2226          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2227                     (ANY_EXTEND:GPI
2228                      (match_operand:ALLX 2 "register_operand" "r")))
2229         (const_int 0)))
2230    (set (match_operand:GPI 0 "register_operand" "=r")
2231         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2232   ""
2233   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2234   [(set_attr "type" "alus_ext")]
2235 )
2236
2237 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2238   [(set (reg:CC_NZ CC_REGNUM)
2239         (compare:CC_NZ
2240          (plus:GPI (ashift:GPI
2241                     (ANY_EXTEND:GPI
2242                      (match_operand:ALLX 1 "register_operand" "r"))
2243                     (match_operand 2 "aarch64_imm3" "Ui3"))
2244                    (match_operand:GPI 3 "register_operand" "rk"))
2245          (const_int 0)))
2246    (set (match_operand:GPI 0 "register_operand" "=rk")
2247         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2248                               (match_dup 2))
2249                   (match_dup 3)))]
2250   ""
2251   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2252   [(set_attr "type" "alus_ext")]
2253 )
2254
2255 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2256   [(set (reg:CC_NZ CC_REGNUM)
2257         (compare:CC_NZ
2258          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2259                     (ashift:GPI
2260                      (ANY_EXTEND:GPI
2261                       (match_operand:ALLX 2 "register_operand" "r"))
2262                      (match_operand 3 "aarch64_imm3" "Ui3")))
2263          (const_int 0)))
2264    (set (match_operand:GPI 0 "register_operand" "=rk")
2265         (minus:GPI (match_dup 1)
2266                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2267                                (match_dup 3))))]
2268   ""
2269   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2270   [(set_attr "type" "alus_ext")]
2271 )
2272
2273 (define_insn "*adds_<optab><mode>_multp2"
2274   [(set (reg:CC_NZ CC_REGNUM)
2275         (compare:CC_NZ
2276          (plus:GPI (ANY_EXTRACT:GPI
2277                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2278                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2279                     (match_operand 3 "const_int_operand" "n")
2280                     (const_int 0))
2281                    (match_operand:GPI 4 "register_operand" "rk"))
2282         (const_int 0)))
2283    (set (match_operand:GPI 0 "register_operand" "=r")
2284         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2285                                    (match_dup 3)
2286                                    (const_int 0))
2287                   (match_dup 4)))]
2288   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2289   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2290   [(set_attr "type" "alus_ext")]
2291 )
2292
2293 (define_insn "*subs_<optab><mode>_multp2"
2294   [(set (reg:CC_NZ CC_REGNUM)
2295         (compare:CC_NZ
2296          (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2297                     (ANY_EXTRACT:GPI
2298                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2299                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2300                      (match_operand 3 "const_int_operand" "n")
2301                      (const_int 0)))
2302         (const_int 0)))
2303    (set (match_operand:GPI 0 "register_operand" "=r")
2304         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2305                                   (mult:GPI (match_dup 1) (match_dup 2))
2306                                   (match_dup 3)
2307                                   (const_int 0))))]
2308   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2309   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2310   [(set_attr "type" "alus_ext")]
2311 )
2312
2313 (define_insn "*add<mode>3nr_compare0"
2314   [(set (reg:CC_NZ CC_REGNUM)
2315         (compare:CC_NZ
2316          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2317                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2318          (const_int 0)))]
2319   ""
2320   "@
2321   cmn\\t%<w>0, %<w>1
2322   cmn\\t%<w>0, %1
2323   cmp\\t%<w>0, #%n1"
2324   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2325 )
2326
2327 (define_insn "aarch64_sub<mode>_compare0"
2328   [(set (reg:CC_NZ CC_REGNUM)
2329         (compare:CC_NZ
2330          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2331                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2332          (const_int 0)))]
2333   ""
2334   "cmp\\t%<w>0, %<w>1"
2335   [(set_attr "type" "alus_sreg")]
2336 )
2337
2338 (define_insn "*compare_neg<mode>"
2339   [(set (reg:CC_Z CC_REGNUM)
2340         (compare:CC_Z
2341          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2342          (match_operand:GPI 1 "register_operand" "r")))]
2343   ""
2344   "cmn\\t%<w>1, %<w>0"
2345   [(set_attr "type" "alus_sreg")]
2346 )
2347
2348 (define_insn "*add_<shift>_<mode>"
2349   [(set (match_operand:GPI 0 "register_operand" "=r")
2350         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2351                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2352                   (match_operand:GPI 3 "register_operand" "r")))]
2353   ""
2354   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2355   [(set_attr "type" "alu_shift_imm")]
2356 )
2357
2358 ;; zero_extend version of above
2359 (define_insn "*add_<shift>_si_uxtw"
2360   [(set (match_operand:DI 0 "register_operand" "=r")
2361         (zero_extend:DI
2362          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2363                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2364                   (match_operand:SI 3 "register_operand" "r"))))]
2365   ""
2366   "add\\t%w0, %w3, %w1, <shift> %2"
2367   [(set_attr "type" "alu_shift_imm")]
2368 )
2369
2370 (define_insn "*add_mul_imm_<mode>"
2371   [(set (match_operand:GPI 0 "register_operand" "=r")
2372         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2373                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2374                   (match_operand:GPI 3 "register_operand" "r")))]
2375   ""
2376   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2377   [(set_attr "type" "alu_shift_imm")]
2378 )
2379
2380 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2381   [(set (match_operand:GPI 0 "register_operand" "=rk")
2382         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2383                   (match_operand:GPI 2 "register_operand" "r")))]
2384   ""
2385   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2386   [(set_attr "type" "alu_ext")]
2387 )
2388
2389 ;; zero_extend version of above
2390 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2391   [(set (match_operand:DI 0 "register_operand" "=rk")
2392         (zero_extend:DI
2393          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2394                   (match_operand:GPI 2 "register_operand" "r"))))]
2395   ""
2396   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2397   [(set_attr "type" "alu_ext")]
2398 )
2399
2400 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2401   [(set (match_operand:GPI 0 "register_operand" "=rk")
2402         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2403                                (match_operand:ALLX 1 "register_operand" "r"))
2404                               (match_operand 2 "aarch64_imm3" "Ui3"))
2405                   (match_operand:GPI 3 "register_operand" "r")))]
2406   ""
2407   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2408   [(set_attr "type" "alu_ext")]
2409 )
2410
2411 ;; zero_extend version of above
2412 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2413   [(set (match_operand:DI 0 "register_operand" "=rk")
2414         (zero_extend:DI
2415          (plus:SI (ashift:SI (ANY_EXTEND:SI
2416                               (match_operand:SHORT 1 "register_operand" "r"))
2417                              (match_operand 2 "aarch64_imm3" "Ui3"))
2418                   (match_operand:SI 3 "register_operand" "r"))))]
2419   ""
2420   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2421   [(set_attr "type" "alu_ext")]
2422 )
2423
2424 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2425   [(set (match_operand:GPI 0 "register_operand" "=rk")
2426         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2427                              (match_operand:ALLX 1 "register_operand" "r"))
2428                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2429                   (match_operand:GPI 3 "register_operand" "r")))]
2430   ""
2431   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2432   [(set_attr "type" "alu_ext")]
2433 )
2434
2435 ;; zero_extend version of above
2436 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2437   [(set (match_operand:DI 0 "register_operand" "=rk")
2438         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2439                              (match_operand:SHORT 1 "register_operand" "r"))
2440                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2441                   (match_operand:SI 3 "register_operand" "r"))))]
2442   ""
2443   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2444   [(set_attr "type" "alu_ext")]
2445 )
2446
2447 (define_insn "*add_<optab><mode>_multp2"
2448   [(set (match_operand:GPI 0 "register_operand" "=rk")
2449         (plus:GPI (ANY_EXTRACT:GPI
2450                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2451                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2452                    (match_operand 3 "const_int_operand" "n")
2453                    (const_int 0))
2454                   (match_operand:GPI 4 "register_operand" "r")))]
2455   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2456   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2457   [(set_attr "type" "alu_ext")]
2458 )
2459
2460 ;; zero_extend version of above
2461 (define_insn "*add_<optab>si_multp2_uxtw"
2462   [(set (match_operand:DI 0 "register_operand" "=rk")
2463         (zero_extend:DI
2464          (plus:SI (ANY_EXTRACT:SI
2465                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2466                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2467                    (match_operand 3 "const_int_operand" "n")
2468                    (const_int 0))
2469                   (match_operand:SI 4 "register_operand" "r"))))]
2470   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2471   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2472   [(set_attr "type" "alu_ext")]
2473 )
2474
2475 (define_expand "add<mode>3_carryin"
2476   [(set (match_operand:GPI 0 "register_operand")
2477         (plus:GPI
2478           (plus:GPI
2479             (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2480             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2481           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2482    ""
2483    ""
2484 )
2485
2486 ;; Note that add with carry with two zero inputs is matched by cset,
2487 ;; and that add with carry with one zero input is matched by cinc.
2488
2489 (define_insn "*add<mode>3_carryin"
2490   [(set (match_operand:GPI 0 "register_operand" "=r")
2491         (plus:GPI
2492           (plus:GPI
2493             (match_operand:GPI 3 "aarch64_carry_operation" "")
2494             (match_operand:GPI 1 "register_operand" "r"))
2495           (match_operand:GPI 2 "register_operand" "r")))]
2496    ""
2497    "adc\\t%<w>0, %<w>1, %<w>2"
2498   [(set_attr "type" "adc_reg")]
2499 )
2500
2501 ;; zero_extend version of above
2502 (define_insn "*addsi3_carryin_uxtw"
2503   [(set (match_operand:DI 0 "register_operand" "=r")
2504         (zero_extend:DI
2505           (plus:SI
2506             (plus:SI
2507               (match_operand:SI 3 "aarch64_carry_operation" "")
2508               (match_operand:SI 1 "register_operand" "r"))
2509             (match_operand:SI 2 "register_operand" "r"))))]
2510    ""
2511    "adc\\t%w0, %w1, %w2"
2512   [(set_attr "type" "adc_reg")]
2513 )
2514
2515 (define_expand "add<mode>3_carryinC"
2516   [(parallel
2517      [(set (match_dup 3)
2518            (compare:CC_ADC
2519              (plus:<DWI>
2520                (plus:<DWI>
2521                  (match_dup 4)
2522                  (zero_extend:<DWI>
2523                    (match_operand:GPI 1 "register_operand")))
2524                (zero_extend:<DWI>
2525                  (match_operand:GPI 2 "register_operand")))
2526              (match_dup 6)))
2527       (set (match_operand:GPI 0 "register_operand")
2528            (plus:GPI
2529              (plus:GPI (match_dup 5) (match_dup 1))
2530              (match_dup 2)))])]
2531    ""
2532 {
2533   operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2534   rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2535   operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2536   operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2537   operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2538                                       << GET_MODE_BITSIZE (<MODE>mode),
2539                                       TImode);
2540 })
2541
2542 (define_insn "*add<mode>3_carryinC_zero"
2543   [(set (reg:CC_ADC CC_REGNUM)
2544         (compare:CC_ADC
2545           (plus:<DWI>
2546             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2547             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2548           (match_operand 4 "const_scalar_int_operand" "")))
2549    (set (match_operand:GPI 0 "register_operand" "=r")
2550         (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2551                   (match_dup 1)))]
2552   "rtx_mode_t (operands[4], <DWI>mode)
2553    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2554    "adcs\\t%<w>0, %<w>1, <w>zr"
2555   [(set_attr "type" "adc_reg")]
2556 )
2557
2558 (define_insn "*add<mode>3_carryinC"
2559   [(set (reg:CC_ADC CC_REGNUM)
2560         (compare:CC_ADC
2561           (plus:<DWI>
2562             (plus:<DWI>
2563               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2564               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2565             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2566           (match_operand 5 "const_scalar_int_operand" "")))
2567    (set (match_operand:GPI 0 "register_operand" "=r")
2568         (plus:GPI
2569           (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2570                     (match_dup 1))
2571           (match_dup 2)))]
2572   "rtx_mode_t (operands[5], <DWI>mode)
2573    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2574    "adcs\\t%<w>0, %<w>1, %<w>2"
2575   [(set_attr "type" "adc_reg")]
2576 )
2577
2578 (define_expand "add<mode>3_carryinV"
2579   [(parallel
2580      [(set (reg:CC_V CC_REGNUM)
2581            (compare:CC_V
2582              (plus:<DWI>
2583                (plus:<DWI>
2584                  (match_dup 3)
2585                  (sign_extend:<DWI>
2586                    (match_operand:GPI 1 "register_operand")))
2587                (sign_extend:<DWI>
2588                  (match_operand:GPI 2 "register_operand")))
2589            (sign_extend:<DWI>
2590              (plus:GPI
2591                (plus:GPI (match_dup 4) (match_dup 1))
2592                (match_dup 2)))))
2593       (set (match_operand:GPI 0 "register_operand")
2594            (plus:GPI
2595              (plus:GPI (match_dup 4) (match_dup 1))
2596              (match_dup 2)))])]
2597    ""
2598 {
2599   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2600   operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2601   operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2602 })
2603
2604 (define_insn "*add<mode>3_carryinV_zero"
2605   [(set (reg:CC_V CC_REGNUM)
2606         (compare:CC_V
2607           (plus:<DWI>
2608             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2609             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2610           (sign_extend:<DWI>
2611             (plus:GPI
2612               (match_operand:GPI 3 "aarch64_carry_operation" "")
2613               (match_dup 1)))))
2614    (set (match_operand:GPI 0 "register_operand" "=r")
2615         (plus:GPI (match_dup 3) (match_dup 1)))]
2616    ""
2617    "adcs\\t%<w>0, %<w>1, <w>zr"
2618   [(set_attr "type" "adc_reg")]
2619 )
2620
2621 (define_insn "*add<mode>3_carryinV"
2622   [(set (reg:CC_V CC_REGNUM)
2623         (compare:CC_V
2624           (plus:<DWI>
2625             (plus:<DWI>
2626               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2627               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2628             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2629           (sign_extend:<DWI>
2630             (plus:GPI
2631               (plus:GPI
2632                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2633                 (match_dup 1))
2634               (match_dup 2)))))
2635    (set (match_operand:GPI 0 "register_operand" "=r")
2636         (plus:GPI
2637           (plus:GPI (match_dup 4) (match_dup 1))
2638           (match_dup 2)))]
2639    ""
2640    "adcs\\t%<w>0, %<w>1, %<w>2"
2641   [(set_attr "type" "adc_reg")]
2642 )
2643
2644 (define_insn "*add_uxt<mode>_shift2"
2645   [(set (match_operand:GPI 0 "register_operand" "=rk")
2646         (plus:GPI (and:GPI
2647                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2648                                (match_operand 2 "aarch64_imm3" "Ui3"))
2649                    (match_operand 3 "const_int_operand" "n"))
2650                   (match_operand:GPI 4 "register_operand" "r")))]
2651   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2652   "*
2653   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2654                                            INTVAL (operands[3])));
2655   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2656   [(set_attr "type" "alu_ext")]
2657 )
2658
2659 ;; zero_extend version of above
2660 (define_insn "*add_uxtsi_shift2_uxtw"
2661   [(set (match_operand:DI 0 "register_operand" "=rk")
2662         (zero_extend:DI
2663          (plus:SI (and:SI
2664                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2665                               (match_operand 2 "aarch64_imm3" "Ui3"))
2666                    (match_operand 3 "const_int_operand" "n"))
2667                   (match_operand:SI 4 "register_operand" "r"))))]
2668   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2669   "*
2670   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2671                                            INTVAL (operands[3])));
2672   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2673   [(set_attr "type" "alu_ext")]
2674 )
2675
2676 (define_insn "*add_uxt<mode>_multp2"
2677   [(set (match_operand:GPI 0 "register_operand" "=rk")
2678         (plus:GPI (and:GPI
2679                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2680                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2681                    (match_operand 3 "const_int_operand" "n"))
2682                   (match_operand:GPI 4 "register_operand" "r")))]
2683   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2684   "*
2685   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2686                                            INTVAL (operands[3])));
2687   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2688   [(set_attr "type" "alu_ext")]
2689 )
2690
2691 ;; zero_extend version of above
2692 (define_insn "*add_uxtsi_multp2_uxtw"
2693   [(set (match_operand:DI 0 "register_operand" "=rk")
2694         (zero_extend:DI
2695          (plus:SI (and:SI
2696                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2697                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2698                    (match_operand 3 "const_int_operand" "n"))
2699                   (match_operand:SI 4 "register_operand" "r"))))]
2700   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2701   "*
2702   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2703                                            INTVAL (operands[3])));
2704   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2705   [(set_attr "type" "alu_ext")]
2706 )
2707
2708 (define_insn "subsi3"
2709   [(set (match_operand:SI 0 "register_operand" "=rk")
2710         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2711                   (match_operand:SI 2 "register_operand" "r")))]
2712   ""
2713   "sub\\t%w0, %w1, %w2"
2714   [(set_attr "type" "alu_sreg")]
2715 )
2716
2717 ;; zero_extend version of above
2718 (define_insn "*subsi3_uxtw"
2719   [(set (match_operand:DI 0 "register_operand" "=rk")
2720         (zero_extend:DI
2721          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2722                    (match_operand:SI 2 "register_operand" "r"))))]
2723   ""
2724   "sub\\t%w0, %w1, %w2"
2725   [(set_attr "type" "alu_sreg")]
2726 )
2727
2728 (define_insn "subdi3"
2729   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2730         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2731                   (match_operand:DI 2 "register_operand" "r,w")))]
2732   ""
2733   "@
2734    sub\\t%x0, %x1, %x2
2735    sub\\t%d0, %d1, %d2"
2736   [(set_attr "type" "alu_sreg, neon_sub")
2737    (set_attr "arch" "*,simd")]
2738 )
2739
2740 (define_expand "subv<GPI:mode>4"
2741   [(match_operand:GPI 0 "register_operand")
2742    (match_operand:GPI 1 "register_operand")
2743    (match_operand:GPI 2 "aarch64_plus_operand")
2744    (label_ref (match_operand 3 "" ""))]
2745   ""
2746 {
2747   if (CONST_INT_P (operands[2]))
2748     emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2749   else
2750     emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2751   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2752
2753   DONE;
2754 })
2755
2756 (define_insn "subv<GPI:mode>_insn"
2757   [(set (reg:CC_V CC_REGNUM)
2758         (compare:CC_V
2759          (sign_extend:<DWI>
2760           (minus:GPI
2761            (match_operand:GPI 1 "register_operand" "rk")
2762            (match_operand:GPI 2 "register_operand" "r")))
2763          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2764                       (sign_extend:<DWI> (match_dup 2)))))
2765    (set (match_operand:GPI 0 "register_operand" "=r")
2766         (minus:GPI (match_dup 1) (match_dup 2)))]
2767   ""
2768   "subs\\t%<w>0, %<w>1, %<w>2"
2769   [(set_attr "type" "alus_sreg")]
2770 )
2771
2772 (define_insn "subv<GPI:mode>_imm"
2773   [(set (reg:CC_V CC_REGNUM)
2774         (compare:CC_V
2775          (sign_extend:<DWI>
2776           (minus:GPI
2777            (match_operand:GPI 1 "register_operand" "rk,rk")
2778            (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2779          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2780                       (match_dup 2))))
2781    (set (match_operand:GPI 0 "register_operand" "=r,r")
2782         (minus:GPI (match_dup 1) (match_dup 2)))]
2783   ""
2784   "@
2785    subs\\t%<w>0, %<w>1, %2
2786    adds\\t%<w>0, %<w>1, #%n2"
2787   [(set_attr "type" "alus_sreg")]
2788 )
2789
2790 (define_expand "negv<GPI:mode>3"
2791   [(match_operand:GPI 0 "register_operand")
2792    (match_operand:GPI 1 "register_operand")
2793    (label_ref (match_operand 2 "" ""))]
2794   ""
2795   {
2796     emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2797     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2798
2799     DONE;
2800   }
2801 )
2802
2803 (define_insn "negv<GPI:mode>_insn"
2804   [(set (reg:CC_V CC_REGNUM)
2805         (compare:CC_V
2806          (sign_extend:<DWI>
2807           (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2808          (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2809    (set (match_operand:GPI 0 "register_operand" "=r")
2810         (neg:GPI (match_dup 1)))]
2811   ""
2812   "negs\\t%<w>0, %<w>1"
2813   [(set_attr "type" "alus_sreg")]
2814 )
2815
2816 (define_insn "negv<GPI:mode>_cmp_only"
2817   [(set (reg:CC_V CC_REGNUM)
2818         (compare:CC_V
2819          (sign_extend:<DWI>
2820           (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2821          (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2822   ""
2823   "negs\\t%<w>zr, %<w>0"
2824   [(set_attr "type" "alus_sreg")]
2825 )
2826
2827 (define_insn "*cmpv<GPI:mode>_insn"
2828   [(set (reg:CC_V CC_REGNUM)
2829         (compare:CC_V
2830          (sign_extend:<DWI>
2831           (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2832                      (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2833          (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2834                     (sign_extend:<DWI> (match_dup 1)))))]
2835   ""
2836   "@
2837    cmp\\t%<w>0, %<w>1
2838    cmp\\t%<w>0, %1
2839    cmp\\t%<w>0, #%n1"
2840   [(set_attr "type" "alus_sreg")]
2841 )
2842
2843 (define_expand "usubv<mode>4"
2844   [(match_operand:GPI 0 "register_operand")
2845    (match_operand:GPI 1 "aarch64_reg_or_zero")
2846    (match_operand:GPI 2 "aarch64_reg_or_zero")
2847    (label_ref (match_operand 3 "" ""))]
2848   ""
2849 {
2850   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2851   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2852
2853   DONE;
2854 })
2855
2856 (define_expand "subti3"
2857   [(set (match_operand:TI 0 "register_operand")
2858         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2859                   (match_operand:TI 2 "register_operand")))]
2860   ""
2861 {
2862   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2863
2864   aarch64_subvti_scratch_regs (operands[1], operands[2],
2865                                &low_dest, &op1_low, &op2_low,
2866                                &high_dest, &op1_high, &op2_high);
2867
2868   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2869   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2870
2871   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2872   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2873   DONE;
2874 })
2875
2876 (define_expand "subvti4"
2877   [(match_operand:TI 0 "register_operand")
2878    (match_operand:TI 1 "register_operand")
2879    (match_operand:TI 2 "aarch64_reg_or_imm")
2880    (label_ref (match_operand 3 "" ""))]
2881   ""
2882 {
2883   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2884
2885   aarch64_subvti_scratch_regs (operands[1], operands[2],
2886                                &low_dest, &op1_low, &op2_low,
2887                                &high_dest, &op1_high, &op2_high);
2888   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2889                          high_dest, op1_high, op2_high, false);
2890
2891   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2892   DONE;
2893 })
2894
2895 (define_expand "usubvti4"
2896   [(match_operand:TI 0 "register_operand")
2897    (match_operand:TI 1 "register_operand")
2898    (match_operand:TI 2 "aarch64_reg_or_imm")
2899    (label_ref (match_operand 3 "" ""))]
2900   ""
2901 {
2902   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2903
2904   aarch64_subvti_scratch_regs (operands[1], operands[2],
2905                                     &low_dest, &op1_low, &op2_low,
2906                                &high_dest, &op1_high, &op2_high);
2907   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2908                          high_dest, op1_high, op2_high, true);
2909
2910   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2911   DONE;
2912 })
2913
2914 (define_expand "negvti3"
2915   [(match_operand:TI 0 "register_operand")
2916    (match_operand:TI 1 "register_operand")
2917    (label_ref (match_operand 2 "" ""))]
2918   ""
2919   {
2920     emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2921                                    gen_lowpart (DImode, operands[1])));
2922     emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2923                                     gen_highpart (DImode, operands[1])));
2924     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2925
2926     DONE;
2927   }
2928 )
2929
2930 (define_insn "negdi_carryout"
2931   [(set (reg:CC CC_REGNUM)
2932         (compare:CC
2933          (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2934    (set (match_operand:DI 0 "register_operand" "=r")
2935         (neg:DI (match_dup 1)))]
2936   ""
2937   "negs\\t%0, %1"
2938   [(set_attr "type" "alus_sreg")]
2939 )
2940
2941 (define_insn "negvdi_carryinV"
2942   [(set (reg:CC_V CC_REGNUM)
2943         (compare:CC_V
2944          (neg:TI (plus:TI
2945                   (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2946                   (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2947          (sign_extend:TI
2948           (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2949                            (match_dup 1))))))
2950    (set (match_operand:DI 0 "register_operand" "=r")
2951         (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2952                          (match_dup 1))))]
2953   ""
2954   "ngcs\\t%0, %1"
2955   [(set_attr "type" "alus_sreg")]
2956 )
2957
2958 (define_insn "*sub<mode>3_compare0"
2959   [(set (reg:CC_NZ CC_REGNUM)
2960         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2961                                   (match_operand:GPI 2 "register_operand" "r"))
2962                        (const_int 0)))
2963    (set (match_operand:GPI 0 "register_operand" "=r")
2964         (minus:GPI (match_dup 1) (match_dup 2)))]
2965   ""
2966   "subs\\t%<w>0, %<w>1, %<w>2"
2967   [(set_attr "type" "alus_sreg")]
2968 )
2969
2970 ;; zero_extend version of above
2971 (define_insn "*subsi3_compare0_uxtw"
2972   [(set (reg:CC_NZ CC_REGNUM)
2973         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
2974                                  (match_operand:SI 2 "register_operand" "r"))
2975                        (const_int 0)))
2976    (set (match_operand:DI 0 "register_operand" "=r")
2977         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2978   ""
2979   "subs\\t%w0, %w1, %w2"
2980   [(set_attr "type" "alus_sreg")]
2981 )
2982
2983 (define_insn "sub<mode>3_compare1_imm"
2984   [(set (reg:CC CC_REGNUM)
2985         (compare:CC
2986           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
2987           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2988    (set (match_operand:GPI 0 "register_operand" "=r,r")
2989         (plus:GPI
2990           (match_dup 1)
2991           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
2992   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
2993   "@
2994   subs\\t%<w>0, %<w>1, %2
2995   adds\\t%<w>0, %<w>1, #%n2"
2996   [(set_attr "type" "alus_imm")]
2997 )
2998
2999 (define_insn "sub<mode>3_compare1"
3000   [(set (reg:CC CC_REGNUM)
3001         (compare:CC
3002           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3003           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3004    (set (match_operand:GPI 0 "register_operand" "=r")
3005         (minus:GPI (match_dup 1) (match_dup 2)))]
3006   ""
3007   "subs\\t%<w>0, %<w>1, %<w>2"
3008   [(set_attr "type" "alus_sreg")]
3009 )
3010
3011 (define_peephole2
3012   [(set (match_operand:GPI 0 "aarch64_general_reg")
3013         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3014                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
3015    (set (reg:CC CC_REGNUM)
3016         (compare:CC
3017           (match_dup 1)
3018           (match_dup 2)))]
3019   "!reg_overlap_mentioned_p (operands[0], operands[1])
3020    && !reg_overlap_mentioned_p (operands[0], operands[2])"
3021   [(const_int 0)]
3022   {
3023     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3024                                          operands[2]));
3025     DONE;
3026   }
3027 )
3028
3029 ;; Same as the above peephole but with the compare and minus in
3030 ;; swapped order.  The restriction on overlap between operand 0
3031 ;; and operands 1 and 2 doesn't apply here.
3032 (define_peephole2
3033   [(set (reg:CC CC_REGNUM)
3034         (compare:CC
3035           (match_operand:GPI 1 "aarch64_reg_or_zero")
3036           (match_operand:GPI 2 "aarch64_reg_or_zero")))
3037    (set (match_operand:GPI 0 "aarch64_general_reg")
3038         (minus:GPI (match_dup 1)
3039                    (match_dup 2)))]
3040   ""
3041   [(const_int 0)]
3042   {
3043     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3044                                          operands[2]));
3045     DONE;
3046   }
3047 )
3048
3049 (define_peephole2
3050   [(set (match_operand:GPI 0 "aarch64_general_reg")
3051         (plus:GPI (match_operand:GPI 1 "register_operand")
3052                   (match_operand:GPI 2 "aarch64_plus_immediate")))
3053    (set (reg:CC CC_REGNUM)
3054         (compare:CC
3055           (match_dup 1)
3056           (match_operand:GPI 3 "const_int_operand")))]
3057   "!reg_overlap_mentioned_p (operands[0], operands[1])
3058    && INTVAL (operands[3]) == -INTVAL (operands[2])"
3059   [(const_int 0)]
3060   {
3061     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3062                                          operands[3], operands[2]));
3063     DONE;
3064   }
3065 )
3066
3067 ;; Same as the above peephole but with the compare and minus in
3068 ;; swapped order.  The restriction on overlap between operand 0
3069 ;; and operands 1 doesn't apply here.
3070 (define_peephole2
3071   [(set (reg:CC CC_REGNUM)
3072         (compare:CC
3073           (match_operand:GPI 1 "register_operand")
3074           (match_operand:GPI 3 "const_int_operand")))
3075    (set (match_operand:GPI 0 "aarch64_general_reg")
3076         (plus:GPI (match_dup 1)
3077                   (match_operand:GPI 2 "aarch64_plus_immediate")))]
3078   "INTVAL (operands[3]) == -INTVAL (operands[2])"
3079   [(const_int 0)]
3080   {
3081     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3082                                          operands[3], operands[2]));
3083     DONE;
3084   }
3085 )
3086
3087 (define_insn "*sub_<shift>_<mode>"
3088   [(set (match_operand:GPI 0 "register_operand" "=r")
3089         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3090                    (ASHIFT:GPI
3091                     (match_operand:GPI 1 "register_operand" "r")
3092                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3093   ""
3094   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3095   [(set_attr "type" "alu_shift_imm")]
3096 )
3097
3098 ;; zero_extend version of above
3099 (define_insn "*sub_<shift>_si_uxtw"
3100   [(set (match_operand:DI 0 "register_operand" "=r")
3101         (zero_extend:DI
3102          (minus:SI (match_operand:SI 3 "register_operand" "r")
3103                    (ASHIFT:SI
3104                     (match_operand:SI 1 "register_operand" "r")
3105                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3106   ""
3107   "sub\\t%w0, %w3, %w1, <shift> %2"
3108   [(set_attr "type" "alu_shift_imm")]
3109 )
3110
3111 (define_insn "*sub_mul_imm_<mode>"
3112   [(set (match_operand:GPI 0 "register_operand" "=r")
3113         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3114                    (mult:GPI
3115                     (match_operand:GPI 1 "register_operand" "r")
3116                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3117   ""
3118   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3119   [(set_attr "type" "alu_shift_imm")]
3120 )
3121
3122 ;; zero_extend version of above
3123 (define_insn "*sub_mul_imm_si_uxtw"
3124   [(set (match_operand:DI 0 "register_operand" "=r")
3125         (zero_extend:DI
3126          (minus:SI (match_operand:SI 3 "register_operand" "r")
3127                    (mult:SI
3128                     (match_operand:SI 1 "register_operand" "r")
3129                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3130   ""
3131   "sub\\t%w0, %w3, %w1, lsl %p2"
3132   [(set_attr "type" "alu_shift_imm")]
3133 )
3134
3135 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3136   [(set (match_operand:GPI 0 "register_operand" "=rk")
3137         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3138                    (ANY_EXTEND:GPI
3139                     (match_operand:ALLX 2 "register_operand" "r"))))]
3140   ""
3141   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3142   [(set_attr "type" "alu_ext")]
3143 )
3144
3145 ;; zero_extend version of above
3146 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3147   [(set (match_operand:DI 0 "register_operand" "=rk")
3148         (zero_extend:DI
3149          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3150                    (ANY_EXTEND:SI
3151                     (match_operand:SHORT 2 "register_operand" "r")))))]
3152   ""
3153   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3154   [(set_attr "type" "alu_ext")]
3155 )
3156
3157 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3158   [(set (match_operand:GPI 0 "register_operand" "=rk")
3159         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3160                    (ashift:GPI (ANY_EXTEND:GPI
3161                                 (match_operand:ALLX 2 "register_operand" "r"))
3162                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
3163   ""
3164   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3165   [(set_attr "type" "alu_ext")]
3166 )
3167
3168 ;; zero_extend version of above
3169 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3170   [(set (match_operand:DI 0 "register_operand" "=rk")
3171         (zero_extend:DI
3172          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3173                    (ashift:SI (ANY_EXTEND:SI
3174                                (match_operand:SHORT 2 "register_operand" "r"))
3175                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
3176   ""
3177   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3178   [(set_attr "type" "alu_ext")]
3179 )
3180
3181 (define_insn "*sub_<optab><mode>_multp2"
3182   [(set (match_operand:GPI 0 "register_operand" "=rk")
3183         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3184                    (ANY_EXTRACT:GPI
3185                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3186                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3187                     (match_operand 3 "const_int_operand" "n")
3188                     (const_int 0))))]
3189   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3190   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3191   [(set_attr "type" "alu_ext")]
3192 )
3193
3194 ;; zero_extend version of above
3195 (define_insn "*sub_<optab>si_multp2_uxtw"
3196   [(set (match_operand:DI 0 "register_operand" "=rk")
3197         (zero_extend:DI
3198          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3199                    (ANY_EXTRACT:SI
3200                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3201                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3202                     (match_operand 3 "const_int_operand" "n")
3203                     (const_int 0)))))]
3204   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3205   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3206   [(set_attr "type" "alu_ext")]
3207 )
3208
3209 ;; The hardware description is op1 + ~op2 + C.
3210 ;;                           = op1 + (-op2 + 1) + (1 - !C)
3211 ;;                           = op1 - op2 - 1 + 1 - !C
3212 ;;                           = op1 - op2 - !C.
3213 ;; We describe the latter.
3214
3215 (define_insn "*sub<mode>3_carryin0"
3216   [(set (match_operand:GPI 0 "register_operand" "=r")
3217         (minus:GPI
3218           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3219           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3220    ""
3221    "sbc\\t%<w>0, %<w>1, <w>zr"
3222   [(set_attr "type" "adc_reg")]
3223 )
3224
3225 ;; zero_extend version of the above
3226 (define_insn "*subsi3_carryin_uxtw"
3227   [(set (match_operand:DI 0 "register_operand" "=r")
3228         (zero_extend:DI
3229           (minus:SI
3230             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3231             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3232    ""
3233    "sbc\\t%w0, %w1, wzr"
3234   [(set_attr "type" "adc_reg")]
3235 )
3236
3237 (define_expand "sub<mode>3_carryin"
3238   [(set (match_operand:GPI 0 "register_operand")
3239         (minus:GPI
3240           (minus:GPI
3241             (match_operand:GPI 1 "aarch64_reg_or_zero")
3242             (match_operand:GPI 2 "register_operand"))
3243           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3244    ""
3245    ""
3246 )
3247
3248 (define_insn "*sub<mode>3_carryin"
3249   [(set (match_operand:GPI 0 "register_operand" "=r")
3250         (minus:GPI
3251           (minus:GPI
3252             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3253             (match_operand:GPI 2 "register_operand" "r"))
3254           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3255
3256    ""
3257    "sbc\\t%<w>0, %<w>1, %<w>2"
3258   [(set_attr "type" "adc_reg")]
3259 )
3260
3261 ;; zero_extend version of the above
3262 (define_insn "*subsi3_carryin_uxtw"
3263   [(set (match_operand:DI 0 "register_operand" "=r")
3264         (zero_extend:DI
3265           (minus:SI
3266             (minus:SI
3267               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3268               (match_operand:SI 2 "register_operand" "r"))
3269             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3270
3271    ""
3272    "sbc\\t%w0, %w1, %w2"
3273   [(set_attr "type" "adc_reg")]
3274 )
3275
3276 (define_insn "*sub<mode>3_carryin_alt"
3277   [(set (match_operand:GPI 0 "register_operand" "=r")
3278         (minus:GPI
3279           (minus:GPI
3280             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3281             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3282           (match_operand:GPI 2 "register_operand" "r")))]
3283    ""
3284    "sbc\\t%<w>0, %<w>1, %<w>2"
3285   [(set_attr "type" "adc_reg")]
3286 )
3287
3288 ;; zero_extend version of the above
3289 (define_insn "*subsi3_carryin_alt_uxtw"
3290   [(set (match_operand:DI 0 "register_operand" "=r")
3291         (zero_extend:DI
3292           (minus:SI
3293             (minus:SI
3294               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3295               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3296             (match_operand:SI 2 "register_operand" "r"))))]
3297    ""
3298    "sbc\\t%w0, %w1, %w2"
3299   [(set_attr "type" "adc_reg")]
3300 )
3301
3302 (define_expand "usub<GPI:mode>3_carryinC"
3303   [(parallel
3304      [(set (reg:CC CC_REGNUM)
3305            (compare:CC
3306              (zero_extend:<DWI>
3307                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3308              (plus:<DWI>
3309                (zero_extend:<DWI>
3310                  (match_operand:GPI 2 "register_operand"))
3311                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3312       (set (match_operand:GPI 0 "register_operand")
3313            (minus:GPI
3314              (minus:GPI (match_dup 1) (match_dup 2))
3315              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3316    ""
3317 )
3318
3319 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3320   [(set (reg:CC CC_REGNUM)
3321         (compare:CC
3322           (const_int 0)
3323           (plus:<DWI>
3324             (zero_extend:<DWI>
3325               (match_operand:GPI 1 "register_operand" "r"))
3326             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3327    (set (match_operand:GPI 0 "register_operand" "=r")
3328         (minus:GPI
3329           (neg:GPI (match_dup 1))
3330           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3331    ""
3332    "sbcs\\t%<w>0, <w>zr, %<w>1"
3333   [(set_attr "type" "adc_reg")]
3334 )
3335
3336 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3337   [(set (reg:CC CC_REGNUM)
3338         (compare:CC
3339           (zero_extend:<DWI>
3340             (match_operand:GPI 1 "register_operand" "r"))
3341           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3342    (set (match_operand:GPI 0 "register_operand" "=r")
3343         (minus:GPI
3344           (match_dup 1)
3345           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3346    ""
3347    "sbcs\\t%<w>0, %<w>1, <w>zr"
3348   [(set_attr "type" "adc_reg")]
3349 )
3350
3351 (define_insn "*usub<GPI:mode>3_carryinC"
3352   [(set (reg:CC CC_REGNUM)
3353         (compare:CC
3354           (zero_extend:<DWI>
3355             (match_operand:GPI 1 "register_operand" "r"))
3356           (plus:<DWI>
3357             (zero_extend:<DWI>
3358               (match_operand:GPI 2 "register_operand" "r"))
3359             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3360    (set (match_operand:GPI 0 "register_operand" "=r")
3361         (minus:GPI
3362           (minus:GPI (match_dup 1) (match_dup 2))
3363           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3364    ""
3365    "sbcs\\t%<w>0, %<w>1, %<w>2"
3366   [(set_attr "type" "adc_reg")]
3367 )
3368
3369 (define_expand "sub<GPI:mode>3_carryinV"
3370   [(parallel
3371      [(set (reg:CC_V CC_REGNUM)
3372            (compare:CC_V
3373             (minus:<DWI>
3374              (sign_extend:<DWI>
3375                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3376              (plus:<DWI>
3377                (sign_extend:<DWI>
3378                  (match_operand:GPI 2 "register_operand"))
3379                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3380             (sign_extend:<DWI>
3381              (minus:GPI (match_dup 1)
3382                         (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3383                                   (match_dup 2))))))
3384       (set (match_operand:GPI 0 "register_operand")
3385            (minus:GPI
3386              (minus:GPI (match_dup 1) (match_dup 2))
3387              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3388    ""
3389 )
3390
3391 (define_insn "*sub<mode>3_carryinV_z2"
3392   [(set (reg:CC_V CC_REGNUM)
3393         (compare:CC_V
3394          (minus:<DWI>
3395           (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3396           (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3397          (sign_extend:<DWI>
3398           (minus:GPI (match_dup 1)
3399                      (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3400    (set (match_operand:GPI 0 "register_operand" "=r")
3401         (minus:GPI
3402          (match_dup 1) (match_dup 3)))]
3403    ""
3404    "sbcs\\t%<w>0, %<w>1, <w>zr"
3405   [(set_attr "type" "adc_reg")]
3406 )
3407
3408 (define_insn "*sub<mode>3_carryinV"
3409   [(set (reg:CC_V CC_REGNUM)
3410         (compare:CC_V
3411          (minus:<DWI>
3412           (sign_extend:<DWI>
3413             (match_operand:GPI 1 "register_operand" "r"))
3414           (plus:<DWI>
3415             (sign_extend:<DWI>
3416               (match_operand:GPI 2 "register_operand" "r"))
3417             (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3418          (sign_extend:<DWI>
3419           (minus:GPI
3420            (match_dup 1)
3421            (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3422                      (match_dup 2))))))
3423    (set (match_operand:GPI 0 "register_operand" "=r")
3424         (minus:GPI
3425           (minus:GPI (match_dup 1) (match_dup 2))
3426           (match_dup 4)))]
3427    ""
3428    "sbcs\\t%<w>0, %<w>1, %<w>2"
3429   [(set_attr "type" "adc_reg")]
3430 )
3431
3432 (define_insn "*sub_uxt<mode>_shift2"
3433   [(set (match_operand:GPI 0 "register_operand" "=rk")
3434         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3435                    (and:GPI
3436                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3437                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3438                     (match_operand 3 "const_int_operand" "n"))))]
3439   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3440   "*
3441   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3442                                            INTVAL (operands[3])));
3443   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3444   [(set_attr "type" "alu_ext")]
3445 )
3446
3447 ;; zero_extend version of above
3448 (define_insn "*sub_uxtsi_shift2_uxtw"
3449   [(set (match_operand:DI 0 "register_operand" "=rk")
3450         (zero_extend:DI
3451          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3452                    (and:SI
3453                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3454                                (match_operand 2 "aarch64_imm3" "Ui3"))
3455                     (match_operand 3 "const_int_operand" "n")))))]
3456   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3457   "*
3458   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3459                                            INTVAL (operands[3])));
3460   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3461   [(set_attr "type" "alu_ext")]
3462 )
3463
3464 (define_insn "*sub_uxt<mode>_multp2"
3465   [(set (match_operand:GPI 0 "register_operand" "=rk")
3466         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3467                    (and:GPI
3468                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3469                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3470                     (match_operand 3 "const_int_operand" "n"))))]
3471   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3472   "*
3473   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3474                                            INTVAL (operands[3])));
3475   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3476   [(set_attr "type" "alu_ext")]
3477 )
3478
3479 ;; zero_extend version of above
3480 (define_insn "*sub_uxtsi_multp2_uxtw"
3481   [(set (match_operand:DI 0 "register_operand" "=rk")
3482         (zero_extend:DI
3483          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3484                    (and:SI
3485                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3486                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3487                     (match_operand 3 "const_int_operand" "n")))))]
3488   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3489   "*
3490   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3491                                            INTVAL (operands[3])));
3492   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3493   [(set_attr "type" "alu_ext")]
3494 )
3495
3496 (define_expand "abs<mode>2"
3497   [(match_operand:GPI 0 "register_operand")
3498    (match_operand:GPI 1 "register_operand")]
3499   ""
3500   {
3501     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3502     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3503     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3504     DONE;
3505   }
3506 )
3507
3508 (define_insn "neg<mode>2"
3509   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3510         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3511   ""
3512   "@
3513    neg\\t%<w>0, %<w>1
3514    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3515   [(set_attr "type" "alu_sreg, neon_neg<q>")
3516    (set_attr "arch" "*,simd")]
3517 )
3518
3519 ;; zero_extend version of above
3520 (define_insn "*negsi2_uxtw"
3521   [(set (match_operand:DI 0 "register_operand" "=r")
3522         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3523   ""
3524   "neg\\t%w0, %w1"
3525   [(set_attr "type" "alu_sreg")]
3526 )
3527
3528 (define_insn "*ngc<mode>"
3529   [(set (match_operand:GPI 0 "register_operand" "=r")
3530         (minus:GPI
3531           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3532           (match_operand:GPI 1 "register_operand" "r")))]
3533   ""
3534   "ngc\\t%<w>0, %<w>1"
3535   [(set_attr "type" "adc_reg")]
3536 )
3537
3538 (define_insn "*ngcsi_uxtw"
3539   [(set (match_operand:DI 0 "register_operand" "=r")
3540         (zero_extend:DI
3541           (minus:SI
3542             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3543             (match_operand:SI 1 "register_operand" "r"))))]
3544   ""
3545   "ngc\\t%w0, %w1"
3546   [(set_attr "type" "adc_reg")]
3547 )
3548
3549 (define_insn "neg<mode>2_compare0"
3550   [(set (reg:CC_NZ CC_REGNUM)
3551         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3552                        (const_int 0)))
3553    (set (match_operand:GPI 0 "register_operand" "=r")
3554         (neg:GPI (match_dup 1)))]
3555   ""
3556   "negs\\t%<w>0, %<w>1"
3557   [(set_attr "type" "alus_sreg")]
3558 )
3559
3560 ;; zero_extend version of above
3561 (define_insn "*negsi2_compare0_uxtw"
3562   [(set (reg:CC_NZ CC_REGNUM)
3563         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3564                        (const_int 0)))
3565    (set (match_operand:DI 0 "register_operand" "=r")
3566         (zero_extend:DI (neg:SI (match_dup 1))))]
3567   ""
3568   "negs\\t%w0, %w1"
3569   [(set_attr "type" "alus_sreg")]
3570 )
3571
3572 (define_insn "*neg_<shift><mode>3_compare0"
3573   [(set (reg:CC_NZ CC_REGNUM)
3574         (compare:CC_NZ
3575          (neg:GPI (ASHIFT:GPI
3576                    (match_operand:GPI 1 "register_operand" "r")
3577                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3578          (const_int 0)))
3579    (set (match_operand:GPI 0 "register_operand" "=r")
3580         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3581   ""
3582   "negs\\t%<w>0, %<w>1, <shift> %2"
3583   [(set_attr "type" "alus_shift_imm")]
3584 )
3585
3586 (define_insn "*neg_<shift>_<mode>2"
3587   [(set (match_operand:GPI 0 "register_operand" "=r")
3588         (neg:GPI (ASHIFT:GPI
3589                   (match_operand:GPI 1 "register_operand" "r")
3590                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3591   ""
3592   "neg\\t%<w>0, %<w>1, <shift> %2"
3593   [(set_attr "type" "alu_shift_imm")]
3594 )
3595
3596 ;; zero_extend version of above
3597 (define_insn "*neg_<shift>_si2_uxtw"
3598   [(set (match_operand:DI 0 "register_operand" "=r")
3599         (zero_extend:DI
3600          (neg:SI (ASHIFT:SI
3601                   (match_operand:SI 1 "register_operand" "r")
3602                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3603   ""
3604   "neg\\t%w0, %w1, <shift> %2"
3605   [(set_attr "type" "alu_shift_imm")]
3606 )
3607
3608 (define_insn "*neg_mul_imm_<mode>2"
3609   [(set (match_operand:GPI 0 "register_operand" "=r")
3610         (neg:GPI (mult:GPI
3611                   (match_operand:GPI 1 "register_operand" "r")
3612                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3613   ""
3614   "neg\\t%<w>0, %<w>1, lsl %p2"
3615   [(set_attr "type" "alu_shift_imm")]
3616 )
3617
3618 ;; zero_extend version of above
3619 (define_insn "*neg_mul_imm_si2_uxtw"
3620   [(set (match_operand:DI 0 "register_operand" "=r")
3621         (zero_extend:DI
3622          (neg:SI (mult:SI
3623                   (match_operand:SI 1 "register_operand" "r")
3624                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3625   ""
3626   "neg\\t%w0, %w1, lsl %p2"
3627   [(set_attr "type" "alu_shift_imm")]
3628 )
3629
3630 (define_insn "mul<mode>3"
3631   [(set (match_operand:GPI 0 "register_operand" "=r")
3632         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3633                   (match_operand:GPI 2 "register_operand" "r")))]
3634   ""
3635   "mul\\t%<w>0, %<w>1, %<w>2"
3636   [(set_attr "type" "mul")]
3637 )
3638
3639 ;; zero_extend version of above
3640 (define_insn "*mulsi3_uxtw"
3641   [(set (match_operand:DI 0 "register_operand" "=r")
3642         (zero_extend:DI
3643          (mult:SI (match_operand:SI 1 "register_operand" "r")
3644                   (match_operand:SI 2 "register_operand" "r"))))]
3645   ""
3646   "mul\\t%w0, %w1, %w2"
3647   [(set_attr "type" "mul")]
3648 )
3649
3650 (define_insn "madd<mode>"
3651   [(set (match_operand:GPI 0 "register_operand" "=r")
3652         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3653                             (match_operand:GPI 2 "register_operand" "r"))
3654                   (match_operand:GPI 3 "register_operand" "r")))]
3655   ""
3656   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3657   [(set_attr "type" "mla")]
3658 )
3659
3660 ;; zero_extend version of above
3661 (define_insn "*maddsi_uxtw"
3662   [(set (match_operand:DI 0 "register_operand" "=r")
3663         (zero_extend:DI
3664          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3665                            (match_operand:SI 2 "register_operand" "r"))
3666                   (match_operand:SI 3 "register_operand" "r"))))]
3667   ""
3668   "madd\\t%w0, %w1, %w2, %w3"
3669   [(set_attr "type" "mla")]
3670 )
3671
3672 (define_insn "*msub<mode>"
3673   [(set (match_operand:GPI 0 "register_operand" "=r")
3674         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3675                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3676                              (match_operand:GPI 2 "register_operand" "r"))))]
3677
3678   ""
3679   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3680   [(set_attr "type" "mla")]
3681 )
3682
3683 ;; zero_extend version of above
3684 (define_insn "*msubsi_uxtw"
3685   [(set (match_operand:DI 0 "register_operand" "=r")
3686         (zero_extend:DI
3687          (minus:SI (match_operand:SI 3 "register_operand" "r")
3688                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3689                             (match_operand:SI 2 "register_operand" "r")))))]
3690
3691   ""
3692   "msub\\t%w0, %w1, %w2, %w3"
3693   [(set_attr "type" "mla")]
3694 )
3695
3696 (define_insn "*mul<mode>_neg"
3697   [(set (match_operand:GPI 0 "register_operand" "=r")
3698         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3699                   (match_operand:GPI 2 "register_operand" "r")))]
3700
3701   ""
3702   "mneg\\t%<w>0, %<w>1, %<w>2"
3703   [(set_attr "type" "mul")]
3704 )
3705
3706 ;; zero_extend version of above
3707 (define_insn "*mulsi_neg_uxtw"
3708   [(set (match_operand:DI 0 "register_operand" "=r")
3709         (zero_extend:DI
3710          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3711                   (match_operand:SI 2 "register_operand" "r"))))]
3712
3713   ""
3714   "mneg\\t%w0, %w1, %w2"
3715   [(set_attr "type" "mul")]
3716 )
3717
3718 (define_insn "<su_optab>mulsidi3"
3719   [(set (match_operand:DI 0 "register_operand" "=r")
3720         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3721                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3722   ""
3723   "<su>mull\\t%0, %w1, %w2"
3724   [(set_attr "type" "<su>mull")]
3725 )
3726
3727 (define_insn "<su_optab>maddsidi4"
3728   [(set (match_operand:DI 0 "register_operand" "=r")
3729         (plus:DI (mult:DI
3730                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3731                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3732                  (match_operand:DI 3 "register_operand" "r")))]
3733   ""
3734   "<su>maddl\\t%0, %w1, %w2, %3"
3735   [(set_attr "type" "<su>mlal")]
3736 )
3737
3738 (define_insn "<su_optab>msubsidi4"
3739   [(set (match_operand:DI 0 "register_operand" "=r")
3740         (minus:DI
3741          (match_operand:DI 3 "register_operand" "r")
3742          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3743                   (ANY_EXTEND:DI
3744                    (match_operand:SI 2 "register_operand" "r")))))]
3745   ""
3746   "<su>msubl\\t%0, %w1, %w2, %3"
3747   [(set_attr "type" "<su>mlal")]
3748 )
3749
3750 (define_insn "*<su_optab>mulsidi_neg"
3751   [(set (match_operand:DI 0 "register_operand" "=r")
3752         (mult:DI (neg:DI
3753                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3754                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3755   ""
3756   "<su>mnegl\\t%0, %w1, %w2"
3757   [(set_attr "type" "<su>mull")]
3758 )
3759
3760 (define_expand "<su_optab>mulditi3"
3761   [(set (match_operand:TI 0 "register_operand")
3762         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3763                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3764   ""
3765 {
3766   rtx low = gen_reg_rtx (DImode);
3767   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3768
3769   rtx high = gen_reg_rtx (DImode);
3770   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3771
3772   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3773   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3774   DONE;
3775 })
3776
3777 ;; The default expansion of multi3 using umuldi3_highpart will perform
3778 ;; the additions in an order that fails to combine into two madd insns.
3779 (define_expand "multi3"
3780   [(set (match_operand:TI 0 "register_operand")
3781         (mult:TI (match_operand:TI 1 "register_operand")
3782                  (match_operand:TI 2 "register_operand")))]
3783   ""
3784 {
3785   rtx l0 = gen_reg_rtx (DImode);
3786   rtx l1 = gen_lowpart (DImode, operands[1]);
3787   rtx l2 = gen_lowpart (DImode, operands[2]);
3788   rtx h0 = gen_reg_rtx (DImode);
3789   rtx h1 = gen_highpart (DImode, operands[1]);
3790   rtx h2 = gen_highpart (DImode, operands[2]);
3791
3792   emit_insn (gen_muldi3 (l0, l1, l2));
3793   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3794   emit_insn (gen_madddi (h0, h1, l2, h0));
3795   emit_insn (gen_madddi (h0, l1, h2, h0));
3796
3797   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3798   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3799   DONE;
3800 })
3801
3802 (define_insn "<su>muldi3_highpart"
3803   [(set (match_operand:DI 0 "register_operand" "=r")
3804         (truncate:DI
3805          (lshiftrt:TI
3806           (mult:TI
3807            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3808            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3809           (const_int 64))))]
3810   ""
3811   "<su>mulh\\t%0, %1, %2"
3812   [(set_attr "type" "<su>mull")]
3813 )
3814
3815 (define_insn "<su_optab>div<mode>3"
3816   [(set (match_operand:GPI 0 "register_operand" "=r")
3817         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3818                      (match_operand:GPI 2 "register_operand" "r")))]
3819   ""
3820   "<su>div\\t%<w>0, %<w>1, %<w>2"
3821   [(set_attr "type" "<su>div")]
3822 )
3823
3824 ;; zero_extend version of above
3825 (define_insn "*<su_optab>divsi3_uxtw"
3826   [(set (match_operand:DI 0 "register_operand" "=r")
3827         (zero_extend:DI
3828          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3829                      (match_operand:SI 2 "register_operand" "r"))))]
3830   ""
3831   "<su>div\\t%w0, %w1, %w2"
3832   [(set_attr "type" "<su>div")]
3833 )
3834
3835 ;; -------------------------------------------------------------------
3836 ;; Comparison insns
3837 ;; -------------------------------------------------------------------
3838
3839 (define_insn "cmp<mode>"
3840   [(set (reg:CC CC_REGNUM)
3841         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3842                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3843   ""
3844   "@
3845    cmp\\t%<w>0, %<w>1
3846    cmp\\t%<w>0, %1
3847    cmn\\t%<w>0, #%n1"
3848   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3849 )
3850
3851 (define_insn "fcmp<mode>"
3852   [(set (reg:CCFP CC_REGNUM)
3853         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3854                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3855    "TARGET_FLOAT"
3856    "@
3857     fcmp\\t%<s>0, #0.0
3858     fcmp\\t%<s>0, %<s>1"
3859   [(set_attr "type" "fcmp<s>")]
3860 )
3861
3862 (define_insn "fcmpe<mode>"
3863   [(set (reg:CCFPE CC_REGNUM)
3864         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3865                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3866    "TARGET_FLOAT"
3867    "@
3868     fcmpe\\t%<s>0, #0.0
3869     fcmpe\\t%<s>0, %<s>1"
3870   [(set_attr "type" "fcmp<s>")]
3871 )
3872
3873 (define_insn "*cmp_swp_<shift>_reg<mode>"
3874   [(set (reg:CC_SWP CC_REGNUM)
3875         (compare:CC_SWP (ASHIFT:GPI
3876                          (match_operand:GPI 0 "register_operand" "r")
3877                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3878                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3879   ""
3880   "cmp\\t%<w>2, %<w>0, <shift> %1"
3881   [(set_attr "type" "alus_shift_imm")]
3882 )
3883
3884 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3885   [(set (reg:CC_SWP CC_REGNUM)
3886         (compare:CC_SWP (ANY_EXTEND:GPI
3887                          (match_operand:ALLX 0 "register_operand" "r"))
3888                         (match_operand:GPI 1 "register_operand" "r")))]
3889   ""
3890   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3891   [(set_attr "type" "alus_ext")]
3892 )
3893
3894 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3895   [(set (reg:CC_SWP CC_REGNUM)
3896         (compare:CC_SWP (ashift:GPI
3897                          (ANY_EXTEND:GPI
3898                           (match_operand:ALLX 0 "register_operand" "r"))
3899                          (match_operand 1 "aarch64_imm3" "Ui3"))
3900         (match_operand:GPI 2 "register_operand" "r")))]
3901   ""
3902   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3903   [(set_attr "type" "alus_ext")]
3904 )
3905
3906 ;; -------------------------------------------------------------------
3907 ;; Store-flag and conditional select insns
3908 ;; -------------------------------------------------------------------
3909
3910 (define_expand "cstore<mode>4"
3911   [(set (match_operand:SI 0 "register_operand")
3912         (match_operator:SI 1 "aarch64_comparison_operator"
3913          [(match_operand:GPI 2 "register_operand")
3914           (match_operand:GPI 3 "aarch64_plus_operand")]))]
3915   ""
3916   "
3917   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3918                                       operands[3]);
3919   operands[3] = const0_rtx;
3920   "
3921 )
3922
3923 (define_expand "cstorecc4"
3924   [(set (match_operand:SI 0 "register_operand")
3925        (match_operator 1 "aarch64_comparison_operator_mode"
3926         [(match_operand 2 "cc_register")
3927          (match_operand 3 "const0_operand")]))]
3928   ""
3929 "{
3930   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3931   DONE;
3932 }")
3933
3934
3935 (define_expand "cstore<mode>4"
3936   [(set (match_operand:SI 0 "register_operand")
3937         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3938          [(match_operand:GPF 2 "register_operand")
3939           (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
3940   ""
3941   "
3942   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3943                                       operands[3]);
3944   operands[3] = const0_rtx;
3945   "
3946 )
3947
3948 (define_insn "aarch64_cstore<mode>"
3949   [(set (match_operand:ALLI 0 "register_operand" "=r")
3950         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3951          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3952   ""
3953   "cset\\t%<w>0, %m1"
3954   [(set_attr "type" "csel")]
3955 )
3956
3957 ;; For a 24-bit immediate CST we can optimize the compare for equality
3958 ;; and branch sequence from:
3959 ;;      mov     x0, #imm1
3960 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3961 ;;      cmp     x1, x0
3962 ;;      cset    x2, <ne,eq>
3963 ;; into the shorter:
3964 ;;      sub     x0, x1, #(CST & 0xfff000)
3965 ;;      subs    x0, x0, #(CST & 0x000fff)
3966 ;;      cset x2, <ne, eq>.
3967 (define_insn_and_split "*compare_cstore<mode>_insn"
3968   [(set (match_operand:GPI 0 "register_operand" "=r")
3969          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3970                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3971    (clobber (reg:CC CC_REGNUM))]
3972   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3973    && !aarch64_plus_operand (operands[2], <MODE>mode)
3974    && !reload_completed"
3975   "#"
3976   "&& true"
3977   [(const_int 0)]
3978   {
3979     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3980     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3981     rtx tmp = gen_reg_rtx (<MODE>mode);
3982     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3983     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3984     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3985     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3986     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3987     DONE;
3988   }
3989   [(set_attr "type" "csel")]
3990 )
3991
3992 ;; zero_extend version of the above
3993 (define_insn "*cstoresi_insn_uxtw"
3994   [(set (match_operand:DI 0 "register_operand" "=r")
3995         (zero_extend:DI
3996          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3997           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3998   ""
3999   "cset\\t%w0, %m1"
4000   [(set_attr "type" "csel")]
4001 )
4002
4003 (define_insn "cstore<mode>_neg"
4004   [(set (match_operand:ALLI 0 "register_operand" "=r")
4005         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4006                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
4007   ""
4008   "csetm\\t%<w>0, %m1"
4009   [(set_attr "type" "csel")]
4010 )
4011
4012 ;; zero_extend version of the above
4013 (define_insn "*cstoresi_neg_uxtw"
4014   [(set (match_operand:DI 0 "register_operand" "=r")
4015         (zero_extend:DI
4016          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4017                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4018   ""
4019   "csetm\\t%w0, %m1"
4020   [(set_attr "type" "csel")]
4021 )
4022
4023 (define_expand "cmov<mode>6"
4024   [(set (match_operand:GPI 0 "register_operand")
4025         (if_then_else:GPI
4026          (match_operator 1 "aarch64_comparison_operator"
4027           [(match_operand:GPI 2 "register_operand")
4028            (match_operand:GPI 3 "aarch64_plus_operand")])
4029          (match_operand:GPI 4 "register_operand")
4030          (match_operand:GPI 5 "register_operand")))]
4031   ""
4032   "
4033   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4034                                       operands[3]);
4035   operands[3] = const0_rtx;
4036   "
4037 )
4038
4039 (define_expand "cmov<mode>6"
4040   [(set (match_operand:GPF 0 "register_operand")
4041         (if_then_else:GPF
4042          (match_operator 1 "aarch64_comparison_operator"
4043           [(match_operand:GPF 2 "register_operand")
4044            (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4045          (match_operand:GPF 4 "register_operand")
4046          (match_operand:GPF 5 "register_operand")))]
4047   ""
4048   "
4049   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4050                                       operands[3]);
4051   operands[3] = const0_rtx;
4052   "
4053 )
4054
4055 (define_insn "*cmov<mode>_insn"
4056   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4057         (if_then_else:ALLI
4058          (match_operator 1 "aarch64_comparison_operator"
4059           [(match_operand 2 "cc_register" "") (const_int 0)])
4060          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4061          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4062   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4063      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4064   ;; Final two alternatives should be unreachable, but included for completeness
4065   "@
4066    csel\\t%<w>0, %<w>3, %<w>4, %m1
4067    csinv\\t%<w>0, %<w>3, <w>zr, %m1
4068    csinv\\t%<w>0, %<w>4, <w>zr, %M1
4069    csinc\\t%<w>0, %<w>3, <w>zr, %m1
4070    csinc\\t%<w>0, %<w>4, <w>zr, %M1
4071    mov\\t%<w>0, -1
4072    mov\\t%<w>0, 1"
4073   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4074 )
4075
4076 ;; zero_extend version of above
4077 (define_insn "*cmovsi_insn_uxtw"
4078   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4079         (zero_extend:DI
4080          (if_then_else:SI
4081           (match_operator 1 "aarch64_comparison_operator"
4082            [(match_operand 2 "cc_register" "") (const_int 0)])
4083           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4084           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4085   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4086      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4087   ;; Final two alternatives should be unreachable, but included for completeness
4088   "@
4089    csel\\t%w0, %w3, %w4, %m1
4090    csinv\\t%w0, %w3, wzr, %m1
4091    csinv\\t%w0, %w4, wzr, %M1
4092    csinc\\t%w0, %w3, wzr, %m1
4093    csinc\\t%w0, %w4, wzr, %M1
4094    mov\\t%w0, -1
4095    mov\\t%w0, 1"
4096   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4097 )
4098
4099 (define_insn "*cmovdi_insn_uxtw"
4100   [(set (match_operand:DI 0 "register_operand" "=r")
4101         (if_then_else:DI
4102          (match_operator 1 "aarch64_comparison_operator"
4103           [(match_operand 2 "cc_register" "") (const_int 0)])
4104          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4105          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4106   ""
4107   "csel\\t%w0, %w3, %w4, %m1"
4108   [(set_attr "type" "csel")]
4109 )
4110
4111 (define_insn "*cmov<mode>_insn"
4112   [(set (match_operand:GPF 0 "register_operand" "=w")
4113         (if_then_else:GPF
4114          (match_operator 1 "aarch64_comparison_operator"
4115           [(match_operand 2 "cc_register" "") (const_int 0)])
4116          (match_operand:GPF 3 "register_operand" "w")
4117          (match_operand:GPF 4 "register_operand" "w")))]
4118   "TARGET_FLOAT"
4119   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4120   [(set_attr "type" "fcsel")]
4121 )
4122
4123 (define_expand "mov<mode>cc"
4124   [(set (match_operand:ALLI 0 "register_operand")
4125         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4126                            (match_operand:ALLI 2 "register_operand")
4127                            (match_operand:ALLI 3 "register_operand")))]
4128   ""
4129   {
4130     rtx ccreg;
4131     enum rtx_code code = GET_CODE (operands[1]);
4132
4133     if (code == UNEQ || code == LTGT)
4134       FAIL;
4135
4136     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4137                                      XEXP (operands[1], 1));
4138     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4139   }
4140 )
4141
4142 (define_expand "mov<GPF:mode><GPI:mode>cc"
4143   [(set (match_operand:GPI 0 "register_operand")
4144         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4145                           (match_operand:GPF 2 "register_operand")
4146                           (match_operand:GPF 3 "register_operand")))]
4147   ""
4148   {
4149     rtx ccreg;
4150     enum rtx_code code = GET_CODE (operands[1]);
4151
4152     if (code == UNEQ || code == LTGT)
4153       FAIL;
4154
4155     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4156                                   XEXP (operands[1], 1));
4157     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4158   }
4159 )
4160
4161 (define_expand "mov<mode>cc"
4162   [(set (match_operand:GPF 0 "register_operand")
4163         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4164                           (match_operand:GPF 2 "register_operand")
4165                           (match_operand:GPF 3 "register_operand")))]
4166   ""
4167   {
4168     rtx ccreg;
4169     enum rtx_code code = GET_CODE (operands[1]);
4170
4171     if (code == UNEQ || code == LTGT)
4172       FAIL;
4173
4174     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4175                                   XEXP (operands[1], 1));
4176     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4177   }
4178 )
4179
4180 (define_expand "<neg_not_op><mode>cc"
4181   [(set (match_operand:GPI 0 "register_operand")
4182         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4183                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4184                           (match_operand:GPI 3 "register_operand")))]
4185   ""
4186   {
4187     rtx ccreg;
4188     enum rtx_code code = GET_CODE (operands[1]);
4189
4190     if (code == UNEQ || code == LTGT)
4191       FAIL;
4192
4193     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4194                                       XEXP (operands[1], 1));
4195     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4196   }
4197 )
4198
4199 ;; CRC32 instructions.
4200 (define_insn "aarch64_<crc_variant>"
4201   [(set (match_operand:SI 0 "register_operand" "=r")
4202         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4203                     (match_operand:<crc_mode> 2 "register_operand" "r")]
4204          CRC))]
4205   "TARGET_CRC32"
4206   {
4207     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4208       return "<crc_variant>\\t%w0, %w1, %x2";
4209     else
4210       return "<crc_variant>\\t%w0, %w1, %w2";
4211   }
4212   [(set_attr "type" "crc")]
4213 )
4214
4215 (define_insn "*csinc2<mode>_insn"
4216   [(set (match_operand:GPI 0 "register_operand" "=r")
4217         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4218                   (match_operand:GPI 1 "register_operand" "r")))]
4219   ""
4220   "cinc\\t%<w>0, %<w>1, %m2"
4221   [(set_attr "type" "csel")]
4222 )
4223
4224 (define_insn "csinc3<mode>_insn"
4225   [(set (match_operand:GPI 0 "register_operand" "=r")
4226         (if_then_else:GPI
4227           (match_operand 1 "aarch64_comparison_operation" "")
4228           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4229                     (const_int 1))
4230           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4231   ""
4232   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4233   [(set_attr "type" "csel")]
4234 )
4235
4236 (define_insn "*csinv3<mode>_insn"
4237   [(set (match_operand:GPI 0 "register_operand" "=r")
4238         (if_then_else:GPI
4239           (match_operand 1 "aarch64_comparison_operation" "")
4240           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4241           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4242   ""
4243   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4244   [(set_attr "type" "csel")]
4245 )
4246
4247 (define_insn "csneg3_uxtw_insn"
4248   [(set (match_operand:DI 0 "register_operand" "=r")
4249         (zero_extend:DI
4250           (if_then_else:SI
4251             (match_operand 1 "aarch64_comparison_operation" "")
4252             (neg:SI (match_operand:SI 2 "register_operand" "r"))
4253             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4254   ""
4255   "csneg\\t%w0, %w3, %w2, %M1"
4256   [(set_attr "type" "csel")]
4257 )
4258
4259 (define_insn "csneg3<mode>_insn"
4260   [(set (match_operand:GPI 0 "register_operand" "=r")
4261         (if_then_else:GPI
4262           (match_operand 1 "aarch64_comparison_operation" "")
4263           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4264           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4265   ""
4266   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4267   [(set_attr "type" "csel")]
4268 )
4269
4270 ;; If X can be loaded by a single CNT[BHWD] instruction,
4271 ;;
4272 ;;    A = UMAX (B, X)
4273 ;;
4274 ;; is equivalent to:
4275 ;;
4276 ;;    TMP = UQDEC[BHWD] (B, X)
4277 ;;    A = TMP + X
4278 ;;
4279 ;; Defining the pattern this way means that:
4280 ;;
4281 ;;    A = UMAX (B, X) - X
4282 ;;
4283 ;; becomes:
4284 ;;
4285 ;;    TMP1 = UQDEC[BHWD] (B, X)
4286 ;;    TMP2 = TMP1 + X
4287 ;;    A = TMP2 - X
4288 ;;
4289 ;; which combine can optimize to:
4290 ;;
4291 ;;    A = UQDEC[BHWD] (B, X)
4292 ;;
4293 ;; We don't use match_operand predicates because the order of the operands
4294 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4295 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4296 (define_expand "umax<mode>3"
4297   [(set (match_operand:GPI 0 "register_operand")
4298         (umax:GPI (match_operand:GPI 1 "")
4299                   (match_operand:GPI 2 "")))]
4300   "TARGET_SVE"
4301   {
4302     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4303       std::swap (operands[1], operands[2]);
4304     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4305       FAIL;
4306     rtx temp = gen_reg_rtx (<MODE>mode);
4307     operands[1] = force_reg (<MODE>mode, operands[1]);
4308     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4309     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4310     DONE;
4311   }
4312 )
4313
4314 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4315 (define_insn "aarch64_uqdec<mode>"
4316   [(set (match_operand:GPI 0 "register_operand" "=r")
4317         (minus:GPI
4318          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4319                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4320          (match_dup 2)))]
4321   "TARGET_SVE"
4322   {
4323     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4324   }
4325 )
4326
4327 ;; -------------------------------------------------------------------
4328 ;; Logical operations
4329 ;; -------------------------------------------------------------------
4330
4331
4332 (define_insn_and_split "*aarch64_and<mode>_imm2"
4333   [(set (match_operand:GPI 0 "register_operand" "=rk")
4334         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4335                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4336   ""
4337   "#"
4338   "true"
4339   [(const_int 0)]
4340   {
4341      HOST_WIDE_INT val = INTVAL (operands[2]);
4342      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4343      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4344
4345      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4346      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4347      DONE;
4348   }
4349 )
4350
4351 (define_insn "<optab><mode>3"
4352   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4353         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4354                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4355   ""
4356   "@
4357   <logical>\\t%<w>0, %<w>1, %<w>2
4358   <logical>\\t%<w>0, %<w>1, %2
4359   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4360   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4361    (set_attr "arch" "*,*,simd")]
4362 )
4363
4364 ;; zero_extend version of above
4365 (define_insn "*<optab>si3_uxtw"
4366   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4367         (zero_extend:DI
4368          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4369                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4370   ""
4371   "@
4372    <logical>\\t%w0, %w1, %w2
4373    <logical>\\t%w0, %w1, %2"
4374   [(set_attr "type" "logic_reg,logic_imm")]
4375 )
4376
4377 (define_insn "*and<mode>3_compare0"
4378   [(set (reg:CC_NZ CC_REGNUM)
4379         (compare:CC_NZ
4380          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4381                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4382          (const_int 0)))
4383    (set (match_operand:GPI 0 "register_operand" "=r,r")
4384         (and:GPI (match_dup 1) (match_dup 2)))]
4385   ""
4386   "@
4387    ands\\t%<w>0, %<w>1, %<w>2
4388    ands\\t%<w>0, %<w>1, %2"
4389   [(set_attr "type" "logics_reg,logics_imm")]
4390 )
4391
4392 ;; zero_extend version of above
4393 (define_insn "*andsi3_compare0_uxtw"
4394   [(set (reg:CC_NZ CC_REGNUM)
4395         (compare:CC_NZ
4396          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4397                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4398          (const_int 0)))
4399    (set (match_operand:DI 0 "register_operand" "=r,r")
4400         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4401   ""
4402   "@
4403    ands\\t%w0, %w1, %w2
4404    ands\\t%w0, %w1, %2"
4405   [(set_attr "type" "logics_reg,logics_imm")]
4406 )
4407
4408 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4409   [(set (reg:CC_NZ CC_REGNUM)
4410         (compare:CC_NZ
4411          (and:GPI (SHIFT:GPI
4412                    (match_operand:GPI 1 "register_operand" "r")
4413                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4414                   (match_operand:GPI 3 "register_operand" "r"))
4415          (const_int 0)))
4416    (set (match_operand:GPI 0 "register_operand" "=r")
4417         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4418   ""
4419   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4420   [(set_attr "type" "logics_shift_imm")]
4421 )
4422
4423 ;; zero_extend version of above
4424 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4425   [(set (reg:CC_NZ CC_REGNUM)
4426         (compare:CC_NZ
4427          (and:SI (SHIFT:SI
4428                   (match_operand:SI 1 "register_operand" "r")
4429                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4430                  (match_operand:SI 3 "register_operand" "r"))
4431          (const_int 0)))
4432    (set (match_operand:DI 0 "register_operand" "=r")
4433         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4434                                 (match_dup 3))))]
4435   ""
4436   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4437   [(set_attr "type" "logics_shift_imm")]
4438 )
4439
4440 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4441   [(set (match_operand:GPI 0 "register_operand" "=r")
4442         (LOGICAL:GPI (SHIFT:GPI
4443                       (match_operand:GPI 1 "register_operand" "r")
4444                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4445                      (match_operand:GPI 3 "register_operand" "r")))]
4446   ""
4447   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4448   [(set_attr "type" "logic_shift_imm")]
4449 )
4450
4451 (define_insn "*<optab>_rol<mode>3"
4452   [(set (match_operand:GPI 0 "register_operand" "=r")
4453         (LOGICAL:GPI (rotate:GPI
4454                       (match_operand:GPI 1 "register_operand" "r")
4455                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4456                      (match_operand:GPI 3 "register_operand" "r")))]
4457   ""
4458   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4459   [(set_attr "type" "logic_shift_imm")]
4460 )
4461
4462 ;; zero_extend versions of above
4463 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4464   [(set (match_operand:DI 0 "register_operand" "=r")
4465         (zero_extend:DI
4466          (LOGICAL:SI (SHIFT:SI
4467                       (match_operand:SI 1 "register_operand" "r")
4468                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4469                      (match_operand:SI 3 "register_operand" "r"))))]
4470   ""
4471   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4472   [(set_attr "type" "logic_shift_imm")]
4473 )
4474
4475 (define_insn "*<optab>_rolsi3_uxtw"
4476   [(set (match_operand:DI 0 "register_operand" "=r")
4477         (zero_extend:DI
4478          (LOGICAL:SI (rotate:SI
4479                       (match_operand:SI 1 "register_operand" "r")
4480                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4481                      (match_operand:SI 3 "register_operand" "r"))))]
4482   ""
4483   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4484   [(set_attr "type" "logic_shift_imm")]
4485 )
4486
4487 (define_insn "one_cmpl<mode>2"
4488   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4489         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4490   ""
4491   "@
4492   mvn\\t%<w>0, %<w>1
4493   mvn\\t%0.8b, %1.8b"
4494   [(set_attr "type" "logic_reg,neon_logic")
4495    (set_attr "arch" "*,simd")]
4496 )
4497
4498 (define_insn "*one_cmpl_<optab><mode>2"
4499   [(set (match_operand:GPI 0 "register_operand" "=r")
4500         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4501                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4502   ""
4503   "mvn\\t%<w>0, %<w>1, <shift> %2"
4504   [(set_attr "type" "logic_shift_imm")]
4505 )
4506
4507 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4508
4509 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4510   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4511         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4512                      (match_operand:GPI 2 "register_operand" "r,w")))]
4513   ""
4514   "@
4515   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4516   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4517   [(set_attr "type" "logic_reg,neon_logic")
4518    (set_attr "arch" "*,simd")]
4519 )
4520
4521 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4522   [(set (match_operand:DI 0 "register_operand" "=r")
4523         (zero_extend:DI
4524           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4525                        (match_operand:SI 2 "register_operand" "r"))))]
4526   ""
4527   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4528   [(set_attr "type" "logic_reg")]
4529 )
4530
4531 (define_insn "*xor_one_cmplsidi3_ze"
4532   [(set (match_operand:DI 0 "register_operand" "=r")
4533         (zero_extend:DI
4534           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4535                           (match_operand:SI 2 "register_operand" "r")))))]
4536   ""
4537   "eon\\t%w0, %w1, %w2"
4538   [(set_attr "type" "logic_reg")]
4539 )
4540
4541 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4542 ;; eon does not operate on SIMD registers so the vector variant must be split.
4543 (define_insn_and_split "*xor_one_cmpl<mode>3"
4544   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4545         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4546                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4547   ""
4548   "@
4549   eon\\t%<w>0, %<w>1, %<w>2
4550   #"
4551   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4552   [(set (match_operand:GPI 0 "register_operand" "=w")
4553         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4554                  (match_operand:GPI 2 "register_operand" "w")))
4555    (set (match_dup 0) (not:GPI (match_dup 0)))]
4556   ""
4557   [(set_attr "type" "logic_reg,multiple")
4558    (set_attr "arch" "*,simd")]
4559 )
4560
4561 (define_insn "*and_one_cmpl<mode>3_compare0"
4562   [(set (reg:CC_NZ CC_REGNUM)
4563         (compare:CC_NZ
4564          (and:GPI (not:GPI
4565                    (match_operand:GPI 1 "register_operand" "r"))
4566                   (match_operand:GPI 2 "register_operand" "r"))
4567          (const_int 0)))
4568    (set (match_operand:GPI 0 "register_operand" "=r")
4569         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4570   ""
4571   "bics\\t%<w>0, %<w>2, %<w>1"
4572   [(set_attr "type" "logics_reg")]
4573 )
4574
4575 ;; zero_extend version of above
4576 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4577   [(set (reg:CC_NZ CC_REGNUM)
4578         (compare:CC_NZ
4579          (and:SI (not:SI
4580                   (match_operand:SI 1 "register_operand" "r"))
4581                  (match_operand:SI 2 "register_operand" "r"))
4582          (const_int 0)))
4583    (set (match_operand:DI 0 "register_operand" "=r")
4584         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4585   ""
4586   "bics\\t%w0, %w2, %w1"
4587   [(set_attr "type" "logics_reg")]
4588 )
4589
4590 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4591   [(set (reg:CC_NZ CC_REGNUM)
4592     (compare:CC_NZ
4593      (and:GPI (not:GPI
4594            (match_operand:GPI 0 "register_operand" "r"))
4595           (match_operand:GPI 1 "register_operand" "r"))
4596      (const_int 0)))]
4597   ""
4598   "bics\\t<w>zr, %<w>1, %<w>0"
4599   [(set_attr "type" "logics_reg")]
4600 )
4601
4602 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4603   [(set (match_operand:GPI 0 "register_operand" "=r")
4604         (LOGICAL:GPI (not:GPI
4605                       (SHIFT:GPI
4606                        (match_operand:GPI 1 "register_operand" "r")
4607                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4608                      (match_operand:GPI 3 "register_operand" "r")))]
4609   ""
4610   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4611   [(set_attr "type" "logic_shift_imm")]
4612 )
4613
4614 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4615   [(set (match_operand:GPI 0 "register_operand" "=r")
4616         (not:GPI (xor:GPI
4617                       (SHIFT:GPI
4618                        (match_operand:GPI 1 "register_operand" "r")
4619                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4620                      (match_operand:GPI 3 "register_operand" "r"))))]
4621   ""
4622   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4623   [(set_attr "type" "logic_shift_imm")]
4624 )
4625
4626 ;; Zero-extend version of the above.
4627 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4628   [(set (match_operand:DI 0 "register_operand" "=r")
4629         (zero_extend:DI
4630           (not:SI (xor:SI
4631                     (SHIFT:SI
4632                       (match_operand:SI 1 "register_operand" "r")
4633                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4634                     (match_operand:SI 3 "register_operand" "r")))))]
4635   ""
4636   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4637   [(set_attr "type" "logic_shift_imm")]
4638 )
4639
4640 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4641   [(set (reg:CC_NZ CC_REGNUM)
4642         (compare:CC_NZ
4643          (and:GPI (not:GPI
4644                    (SHIFT:GPI
4645                     (match_operand:GPI 1 "register_operand" "r")
4646                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4647                   (match_operand:GPI 3 "register_operand" "r"))
4648          (const_int 0)))
4649    (set (match_operand:GPI 0 "register_operand" "=r")
4650         (and:GPI (not:GPI
4651                   (SHIFT:GPI
4652                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4653   ""
4654   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4655   [(set_attr "type" "logics_shift_imm")]
4656 )
4657
4658 ;; zero_extend version of above
4659 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4660   [(set (reg:CC_NZ CC_REGNUM)
4661         (compare:CC_NZ
4662          (and:SI (not:SI
4663                   (SHIFT:SI
4664                    (match_operand:SI 1 "register_operand" "r")
4665                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4666                  (match_operand:SI 3 "register_operand" "r"))
4667          (const_int 0)))
4668    (set (match_operand:DI 0 "register_operand" "=r")
4669         (zero_extend:DI (and:SI
4670                          (not:SI
4671                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4672   ""
4673   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4674   [(set_attr "type" "logics_shift_imm")]
4675 )
4676
4677 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4678   [(set (reg:CC_NZ CC_REGNUM)
4679     (compare:CC_NZ
4680      (and:GPI (not:GPI
4681            (SHIFT:GPI
4682             (match_operand:GPI 0 "register_operand" "r")
4683             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4684           (match_operand:GPI 2 "register_operand" "r"))
4685      (const_int 0)))]
4686   ""
4687   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4688   [(set_attr "type" "logics_shift_imm")]
4689 )
4690
4691 (define_insn "clz<mode>2"
4692   [(set (match_operand:GPI 0 "register_operand" "=r")
4693         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4694   ""
4695   "clz\\t%<w>0, %<w>1"
4696   [(set_attr "type" "clz")]
4697 )
4698
4699 (define_expand "ffs<mode>2"
4700   [(match_operand:GPI 0 "register_operand")
4701    (match_operand:GPI 1 "register_operand")]
4702   ""
4703   {
4704     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4705     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4706
4707     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4708     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4709     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4710     DONE;
4711   }
4712 )
4713
4714 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4715 ;;
4716 ;; MOV  v.1d, x0
4717 ;; CNT  v1.8b, v.8b
4718 ;; ADDV b2, v1.8b
4719 ;; MOV  w0, v2.b[0]
4720
4721 (define_expand "popcount<mode>2"
4722   [(match_operand:GPI 0 "register_operand")
4723    (match_operand:GPI 1 "register_operand")]
4724   "TARGET_SIMD"
4725 {
4726   rtx v = gen_reg_rtx (V8QImode);
4727   rtx v1 = gen_reg_rtx (V8QImode);
4728   rtx r = gen_reg_rtx (QImode);
4729   rtx in = operands[1];
4730   rtx out = operands[0];
4731   if(<MODE>mode == SImode)
4732     {
4733       rtx tmp;
4734       tmp = gen_reg_rtx (DImode);
4735       /* If we have SImode, zero extend to DImode, pop count does
4736          not change if we have extra zeros. */
4737       emit_insn (gen_zero_extendsidi2 (tmp, in));
4738       in = tmp;
4739     }
4740   emit_move_insn (v, gen_lowpart (V8QImode, in));
4741   emit_insn (gen_popcountv8qi2 (v1, v));
4742   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4743   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4744   DONE;
4745 })
4746
4747 (define_insn "clrsb<mode>2"
4748   [(set (match_operand:GPI 0 "register_operand" "=r")
4749         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4750   ""
4751   "cls\\t%<w>0, %<w>1"
4752   [(set_attr "type" "clz")]
4753 )
4754
4755 (define_insn "rbit<mode>2"
4756   [(set (match_operand:GPI 0 "register_operand" "=r")
4757         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4758   ""
4759   "rbit\\t%<w>0, %<w>1"
4760   [(set_attr "type" "rbit")]
4761 )
4762
4763 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4764 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4765 ;; expression and split after reload to enable scheduling them apart if
4766 ;; needed.
4767
4768 (define_insn_and_split "ctz<mode>2"
4769  [(set (match_operand:GPI           0 "register_operand" "=r")
4770        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4771   ""
4772   "#"
4773   "reload_completed"
4774   [(const_int 0)]
4775   "
4776   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4777   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4778   DONE;
4779 ")
4780
4781 (define_insn "*and<mode>_compare0"
4782   [(set (reg:CC_NZ CC_REGNUM)
4783         (compare:CC_NZ
4784          (match_operand:SHORT 0 "register_operand" "r")
4785          (const_int 0)))]
4786   ""
4787   "tst\\t%<w>0, <short_mask>"
4788   [(set_attr "type" "alus_imm")]
4789 )
4790
4791 (define_insn "*ands<GPI:mode>_compare0"
4792   [(set (reg:CC_NZ CC_REGNUM)
4793         (compare:CC_NZ
4794          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4795          (const_int 0)))
4796    (set (match_operand:GPI 0 "register_operand" "=r")
4797         (zero_extend:GPI (match_dup 1)))]
4798   ""
4799   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4800   [(set_attr "type" "alus_imm")]
4801 )
4802
4803 (define_insn "*and<mode>3nr_compare0"
4804   [(set (reg:CC_NZ CC_REGNUM)
4805         (compare:CC_NZ
4806          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4807                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4808          (const_int 0)))]
4809   ""
4810   "@
4811    tst\\t%<w>0, %<w>1
4812    tst\\t%<w>0, %1"
4813   [(set_attr "type" "logics_reg,logics_imm")]
4814 )
4815
4816 (define_split
4817   [(set (reg:CC_NZ CC_REGNUM)
4818         (compare:CC_NZ
4819          (and:GPI (match_operand:GPI 0 "register_operand")
4820                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4821          (const_int 0)))
4822    (clobber (match_operand:SI 2 "register_operand"))]
4823   ""
4824   [(set (match_dup 2) (match_dup 1))
4825    (set (reg:CC_NZ CC_REGNUM)
4826         (compare:CC_NZ
4827          (and:GPI (match_dup 0)
4828                   (match_dup 2))
4829          (const_int 0)))]
4830 )
4831
4832 (define_insn "*and<mode>3nr_compare0_zextract"
4833   [(set (reg:CC_NZ CC_REGNUM)
4834         (compare:CC_NZ
4835          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4836                   (match_operand:GPI 1 "const_int_operand" "n")
4837                   (match_operand:GPI 2 "const_int_operand" "n"))
4838          (const_int 0)))]
4839   "INTVAL (operands[1]) > 0
4840    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4841         <= GET_MODE_BITSIZE (<MODE>mode))
4842    && aarch64_bitmask_imm (
4843         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4844                                                  operands[2])),
4845         <MODE>mode)"
4846   {
4847     operands[1]
4848       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4849     return "tst\\t%<w>0, %1";
4850   }
4851   [(set_attr "type" "logics_shift_imm")]
4852 )
4853
4854 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4855   [(set (reg:CC_NZ CC_REGNUM)
4856         (compare:CC_NZ
4857          (and:GPI (SHIFT:GPI
4858                    (match_operand:GPI 0 "register_operand" "r")
4859                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4860                   (match_operand:GPI 2 "register_operand" "r"))
4861         (const_int 0)))]
4862   ""
4863   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4864   [(set_attr "type" "logics_shift_imm")]
4865 )
4866
4867 (define_split
4868   [(set (reg:CC_NZ CC_REGNUM)
4869         (compare:CC_NZ
4870          (and:GPI (SHIFT:GPI
4871                    (match_operand:GPI 0 "register_operand")
4872                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4873                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4874         (const_int 0)))
4875     (clobber (match_operand:SI 3 "register_operand"))]
4876   ""
4877   [(set (match_dup 3) (match_dup 2))
4878    (set (reg:CC_NZ CC_REGNUM)
4879         (compare:CC_NZ
4880          (and:GPI (SHIFT:GPI
4881                    (match_dup 0)
4882                    (match_dup 1))
4883                   (match_dup 3))
4884          (const_int 0)))]
4885 )
4886
4887 ;; -------------------------------------------------------------------
4888 ;; Shifts
4889 ;; -------------------------------------------------------------------
4890
4891 (define_expand "<optab><mode>3"
4892   [(set (match_operand:GPI 0 "register_operand")
4893         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4894                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4895   ""
4896   {
4897     if (CONST_INT_P (operands[2]))
4898       {
4899         operands[2] = GEN_INT (INTVAL (operands[2])
4900                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4901
4902         if (operands[2] == const0_rtx)
4903           {
4904             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4905             DONE;
4906           }
4907       }
4908   }
4909 )
4910
4911 (define_expand "ashl<mode>3"
4912   [(set (match_operand:SHORT 0 "register_operand")
4913         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4914                       (match_operand:QI 2 "const_int_operand")))]
4915   ""
4916   {
4917     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4918
4919     if (operands[2] == const0_rtx)
4920       {
4921         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4922         DONE;
4923       }
4924   }
4925 )
4926
4927 (define_expand "rotr<mode>3"
4928   [(set (match_operand:GPI 0 "register_operand")
4929         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4930                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4931   ""
4932   {
4933     if (CONST_INT_P (operands[2]))
4934       {
4935         operands[2] = GEN_INT (INTVAL (operands[2])
4936                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4937
4938         if (operands[2] == const0_rtx)
4939           {
4940             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4941             DONE;
4942           }
4943       }
4944   }
4945 )
4946
4947 (define_expand "rotl<mode>3"
4948   [(set (match_operand:GPI 0 "register_operand")
4949         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4950                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4951   ""
4952   {
4953     /* (SZ - cnt) % SZ == -cnt % SZ */
4954     if (CONST_INT_P (operands[2]))
4955       {
4956         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4957                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4958         if (operands[2] == const0_rtx)
4959           {
4960             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4961             DONE;
4962           }
4963       }
4964     else
4965       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4966                                         NULL_RTX, 1);
4967   }
4968 )
4969
4970 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4971 ;; they truncate the shift/rotate amount by the size of the registers they
4972 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
4973 ;; such redundant masking instructions.  GCC can do that automatically when
4974 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4975 ;; because some of the SISD shift alternatives don't perform this truncations.
4976 ;; So this pattern exists to catch such cases.
4977
4978 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4979   [(set (match_operand:GPI 0 "register_operand" "=r")
4980         (SHIFT:GPI
4981           (match_operand:GPI 1 "register_operand" "r")
4982           (match_operator 4 "subreg_lowpart_operator"
4983            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4984                      (match_operand 3 "const_int_operand" "n"))])))]
4985   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4986   "<shift>\t%<w>0, %<w>1, %<w>2"
4987   [(set_attr "type" "shift_reg")]
4988 )
4989
4990 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4991   [(set (match_operand:GPI 0 "register_operand" "=&r")
4992         (SHIFT:GPI
4993           (match_operand:GPI 1 "register_operand" "r")
4994           (match_operator 4 "subreg_lowpart_operator"
4995           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4996                            (match_operand 3 "const_int_operand" "n")))])))]
4997   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4998   "#"
4999   "&& true"
5000   [(const_int 0)]
5001   {
5002     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5003                : lowpart_subreg (SImode, operands[0], <MODE>mode));
5004     emit_insn (gen_negsi2 (tmp, operands[2]));
5005
5006     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5007     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5008                                      SUBREG_BYTE (operands[4]));
5009     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5010     DONE;
5011   }
5012 )
5013
5014 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5015   [(set (match_operand:GPI 0 "register_operand" "=&r")
5016         (ashift:GPI
5017           (match_operand:GPI 1 "register_operand" "r")
5018           (minus:QI (match_operand 2 "const_int_operand" "n")
5019                     (match_operator 5 "subreg_lowpart_operator"
5020                     [(and:SI (match_operand:SI 3 "register_operand" "r")
5021                              (match_operand 4 "const_int_operand" "n"))]))))]
5022   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5023    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5024   "#"
5025   "&& true"
5026   [(const_int 0)]
5027   {
5028     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5029                : operands[0]);
5030
5031     emit_insn (gen_negsi2 (tmp, operands[3]));
5032
5033     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5034     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5035                                      SUBREG_BYTE (operands[5]));
5036
5037     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5038     DONE;
5039   }
5040 )
5041
5042 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5043   [(set (match_operand:DI 0 "register_operand" "=r")
5044         (SHIFT:DI
5045           (match_operand:DI 1 "register_operand" "r")
5046           (match_operator 4 "subreg_lowpart_operator"
5047            [(and:SI (match_operand:SI 2 "register_operand" "r")
5048                     (match_operand 3 "const_int_operand" "n"))])))]
5049   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5050 {
5051   rtx xop[3];
5052   xop[0] = operands[0];
5053   xop[1] = operands[1];
5054   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5055   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5056   return "";
5057 }
5058   [(set_attr "type" "shift_reg")]
5059 )
5060
5061 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5062   [(set (match_operand:GPI 0 "register_operand" "=&r")
5063         (ASHIFT:GPI
5064           (match_operand:GPI 1 "register_operand" "r")
5065           (minus:QI (match_operand 2 "const_int_operand" "n")
5066                     (match_operand:QI 3 "register_operand" "r"))))]
5067   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5068   "#"
5069   "&& true"
5070   [(const_int 0)]
5071   {
5072     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5073
5074     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5075                : gen_lowpart (SImode, operands[0]));
5076
5077     emit_insn (gen_negsi2 (tmp, subreg_tmp));
5078
5079     rtx and_op = gen_rtx_AND (SImode, tmp,
5080                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5081
5082     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5083
5084     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5085     DONE;
5086   }
5087   [(set_attr "length" "8")]
5088 )
5089
5090 ;; Logical left shift using SISD or Integer instruction
5091 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5092   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5093         (ashift:GPI
5094           (match_operand:GPI 1 "register_operand" "r,r,w,w")
5095           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5096   ""
5097   "@
5098    lsl\t%<w>0, %<w>1, %2
5099    lsl\t%<w>0, %<w>1, %<w>2
5100    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5101    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5102   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5103    (set_attr "arch" "*,*,simd,simd")]
5104 )
5105
5106 ;; Logical right shift using SISD or Integer instruction
5107 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5108   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5109         (lshiftrt:GPI
5110          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5111          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5112                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5113   ""
5114   "@
5115    lsr\t%<w>0, %<w>1, %2
5116    lsr\t%<w>0, %<w>1, %<w>2
5117    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5118    #
5119    #"
5120   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5121    (set_attr "arch" "*,*,simd,simd,simd")]
5122 )
5123
5124 (define_split
5125   [(set (match_operand:DI 0 "aarch64_simd_register")
5126         (lshiftrt:DI
5127            (match_operand:DI 1 "aarch64_simd_register")
5128            (match_operand:QI 2 "aarch64_simd_register")))]
5129   "TARGET_SIMD && reload_completed"
5130   [(set (match_dup 3)
5131         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5132    (set (match_dup 0)
5133         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5134   {
5135     operands[3] = gen_lowpart (QImode, operands[0]);
5136   }
5137 )
5138
5139 (define_split
5140   [(set (match_operand:SI 0 "aarch64_simd_register")
5141         (lshiftrt:SI
5142            (match_operand:SI 1 "aarch64_simd_register")
5143            (match_operand:QI 2 "aarch64_simd_register")))]
5144   "TARGET_SIMD && reload_completed"
5145   [(set (match_dup 3)
5146         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5147    (set (match_dup 0)
5148         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5149   {
5150     operands[3] = gen_lowpart (QImode, operands[0]);
5151   }
5152 )
5153
5154 ;; Arithmetic right shift using SISD or Integer instruction
5155 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5156   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5157         (ashiftrt:GPI
5158           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5159           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5160                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5161   ""
5162   "@
5163    asr\t%<w>0, %<w>1, %2
5164    asr\t%<w>0, %<w>1, %<w>2
5165    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5166    #
5167    #"
5168   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5169    (set_attr "arch" "*,*,simd,simd,simd")]
5170 )
5171
5172 (define_split
5173   [(set (match_operand:DI 0 "aarch64_simd_register")
5174         (ashiftrt:DI
5175            (match_operand:DI 1 "aarch64_simd_register")
5176            (match_operand:QI 2 "aarch64_simd_register")))]
5177   "TARGET_SIMD && reload_completed"
5178   [(set (match_dup 3)
5179         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5180    (set (match_dup 0)
5181         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5182 {
5183   operands[3] = gen_lowpart (QImode, operands[0]);
5184 }
5185 )
5186
5187 (define_split
5188   [(set (match_operand:SI 0 "aarch64_simd_register")
5189         (ashiftrt:SI
5190            (match_operand:SI 1 "aarch64_simd_register")
5191            (match_operand:QI 2 "aarch64_simd_register")))]
5192   "TARGET_SIMD && reload_completed"
5193   [(set (match_dup 3)
5194         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5195    (set (match_dup 0)
5196         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5197 {
5198   operands[3] = gen_lowpart (QImode, operands[0]);
5199 }
5200 )
5201
5202 (define_insn "*aarch64_sisd_ushl"
5203   [(set (match_operand:DI 0 "register_operand" "=w")
5204         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5205                     (match_operand:QI 2 "register_operand" "w")]
5206                    UNSPEC_SISD_USHL))]
5207   "TARGET_SIMD"
5208   "ushl\t%d0, %d1, %d2"
5209   [(set_attr "type" "neon_shift_reg")]
5210 )
5211
5212 (define_insn "*aarch64_ushl_2s"
5213   [(set (match_operand:SI 0 "register_operand" "=w")
5214         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5215                     (match_operand:QI 2 "register_operand" "w")]
5216                    UNSPEC_USHL_2S))]
5217   "TARGET_SIMD"
5218   "ushl\t%0.2s, %1.2s, %2.2s"
5219   [(set_attr "type" "neon_shift_reg")]
5220 )
5221
5222 (define_insn "*aarch64_sisd_sshl"
5223   [(set (match_operand:DI 0 "register_operand" "=w")
5224         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5225                     (match_operand:QI 2 "register_operand" "w")]
5226                    UNSPEC_SISD_SSHL))]
5227   "TARGET_SIMD"
5228   "sshl\t%d0, %d1, %d2"
5229   [(set_attr "type" "neon_shift_reg")]
5230 )
5231
5232 (define_insn "*aarch64_sshl_2s"
5233   [(set (match_operand:SI 0 "register_operand" "=w")
5234         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5235                     (match_operand:QI 2 "register_operand" "w")]
5236                    UNSPEC_SSHL_2S))]
5237   "TARGET_SIMD"
5238   "sshl\t%0.2s, %1.2s, %2.2s"
5239   [(set_attr "type" "neon_shift_reg")]
5240 )
5241
5242 (define_insn "*aarch64_sisd_neg_qi"
5243   [(set (match_operand:QI 0 "register_operand" "=w")
5244         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5245                    UNSPEC_SISD_NEG))]
5246   "TARGET_SIMD"
5247   "neg\t%d0, %d1"
5248   [(set_attr "type" "neon_neg")]
5249 )
5250
5251 ;; Rotate right
5252 (define_insn "*ror<mode>3_insn"
5253   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5254      (rotatert:GPI
5255        (match_operand:GPI 1 "register_operand" "r,r")
5256        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5257   ""
5258   "@
5259    ror\\t%<w>0, %<w>1, %2
5260    ror\\t%<w>0, %<w>1, %<w>2"
5261   [(set_attr "type" "rotate_imm,shift_reg")]
5262 )
5263
5264 ;; zero_extend version of above
5265 (define_insn "*<optab>si3_insn_uxtw"
5266   [(set (match_operand:DI 0 "register_operand" "=r,r")
5267         (zero_extend:DI (SHIFT:SI
5268          (match_operand:SI 1 "register_operand" "r,r")
5269          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5270   ""
5271   "@
5272    <shift>\\t%w0, %w1, %2
5273    <shift>\\t%w0, %w1, %w2"
5274   [(set_attr "type" "bfx,shift_reg")]
5275 )
5276
5277 (define_insn "*<optab><mode>3_insn"
5278   [(set (match_operand:SHORT 0 "register_operand" "=r")
5279         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5280                       (match_operand 2 "const_int_operand" "n")))]
5281   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5282 {
5283   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5284   return "<bfshift>\t%w0, %w1, %2, %3";
5285 }
5286   [(set_attr "type" "bfx")]
5287 )
5288
5289 (define_insn "*extr<mode>5_insn"
5290   [(set (match_operand:GPI 0 "register_operand" "=r")
5291         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5292                              (match_operand 3 "const_int_operand" "n"))
5293                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5294                                (match_operand 4 "const_int_operand" "n"))))]
5295   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5296    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5297   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5298   [(set_attr "type" "rotate_imm")]
5299 )
5300
5301 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5302 ;; so we have to match both orderings.
5303 (define_insn "*extr<mode>5_insn_alt"
5304   [(set (match_operand:GPI 0 "register_operand" "=r")
5305         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5306                                 (match_operand 4 "const_int_operand" "n"))
5307                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5308                               (match_operand 3 "const_int_operand" "n"))))]
5309   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5310    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5311        == GET_MODE_BITSIZE (<MODE>mode))"
5312   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5313   [(set_attr "type" "rotate_imm")]
5314 )
5315
5316 ;; zero_extend version of the above
5317 (define_insn "*extrsi5_insn_uxtw"
5318   [(set (match_operand:DI 0 "register_operand" "=r")
5319         (zero_extend:DI
5320          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5321                             (match_operand 3 "const_int_operand" "n"))
5322                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5323                               (match_operand 4 "const_int_operand" "n")))))]
5324   "UINTVAL (operands[3]) < 32 &&
5325    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5326   "extr\\t%w0, %w1, %w2, %4"
5327   [(set_attr "type" "rotate_imm")]
5328 )
5329
5330 (define_insn "*extrsi5_insn_uxtw_alt"
5331   [(set (match_operand:DI 0 "register_operand" "=r")
5332         (zero_extend:DI
5333          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5334                                (match_operand 4 "const_int_operand" "n"))
5335                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5336                             (match_operand 3 "const_int_operand" "n")))))]
5337   "UINTVAL (operands[3]) < 32 &&
5338    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5339   "extr\\t%w0, %w1, %w2, %4"
5340   [(set_attr "type" "rotate_imm")]
5341 )
5342
5343 (define_insn "*ror<mode>3_insn"
5344   [(set (match_operand:GPI 0 "register_operand" "=r")
5345         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5346                     (match_operand 2 "const_int_operand" "n")))]
5347   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5348 {
5349   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5350   return "ror\\t%<w>0, %<w>1, %3";
5351 }
5352   [(set_attr "type" "rotate_imm")]
5353 )
5354
5355 ;; zero_extend version of the above
5356 (define_insn "*rorsi3_insn_uxtw"
5357   [(set (match_operand:DI 0 "register_operand" "=r")
5358         (zero_extend:DI
5359          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5360                     (match_operand 2 "const_int_operand" "n"))))]
5361   "UINTVAL (operands[2]) < 32"
5362 {
5363   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5364   return "ror\\t%w0, %w1, %3";
5365 }
5366   [(set_attr "type" "rotate_imm")]
5367 )
5368
5369 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5370   [(set (match_operand:GPI 0 "register_operand" "=r")
5371         (ANY_EXTEND:GPI
5372          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5373                        (match_operand 2 "const_int_operand" "n"))))]
5374   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5375 {
5376   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5377   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5378 }
5379   [(set_attr "type" "bfx")]
5380 )
5381
5382 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5383   [(set (match_operand:GPI 0 "register_operand" "=r")
5384         (zero_extend:GPI
5385          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5386                          (match_operand 2 "const_int_operand" "n"))))]
5387   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5388 {
5389   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5390   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5391 }
5392   [(set_attr "type" "bfx")]
5393 )
5394
5395 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5396   [(set (match_operand:GPI 0 "register_operand" "=r")
5397         (sign_extend:GPI
5398          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5399                          (match_operand 2 "const_int_operand" "n"))))]
5400   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5401 {
5402   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5403   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5404 }
5405   [(set_attr "type" "bfx")]
5406 )
5407
5408 ;; -------------------------------------------------------------------
5409 ;; Bitfields
5410 ;; -------------------------------------------------------------------
5411
5412 (define_expand "<optab>"
5413   [(set (match_operand:DI 0 "register_operand")
5414         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5415                         (match_operand 2
5416                           "aarch64_simd_shift_imm_offset_di")
5417                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5418   ""
5419   {
5420     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5421                    1, GET_MODE_BITSIZE (DImode) - 1))
5422      FAIL;
5423   }
5424 )
5425
5426
5427 (define_insn "*<optab><mode>"
5428   [(set (match_operand:GPI 0 "register_operand" "=r")
5429         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5430                          (match_operand 2
5431                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5432                          (match_operand 3
5433                            "aarch64_simd_shift_imm_<mode>" "n")))]
5434   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5435              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5436   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5437   [(set_attr "type" "bfx")]
5438 )
5439
5440 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5441 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5442 (define_split
5443   [(set (match_operand:DI 0 "register_operand")
5444         (zero_extract:DI (match_operand:DI 1 "register_operand")
5445                          (match_operand 2
5446                            "aarch64_simd_shift_imm_offset_di")
5447                          (match_operand 3
5448                            "aarch64_simd_shift_imm_di")))]
5449   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5450              GET_MODE_BITSIZE (DImode) - 1)
5451    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5452        == GET_MODE_BITSIZE (SImode)"
5453   [(set (match_dup 0)
5454         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5455   {
5456     operands[4] = gen_lowpart (SImode, operands[1]);
5457   }
5458 )
5459
5460 ;; Bitfield Insert (insv)
5461 (define_expand "insv<mode>"
5462   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5463                           (match_operand 1 "const_int_operand")
5464                           (match_operand 2 "const_int_operand"))
5465         (match_operand:GPI 3 "general_operand"))]
5466   ""
5467 {
5468   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5469   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5470   rtx value = operands[3];
5471
5472   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5473     FAIL;
5474
5475   if (CONST_INT_P (value))
5476     {
5477       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5478
5479       /* Prefer AND/OR for inserting all zeros or all ones.  */
5480       if ((UINTVAL (value) & mask) == 0
5481            || (UINTVAL (value) & mask) == mask)
5482         FAIL;
5483
5484       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5485       if (width == 16 && (pos % 16) == 0)
5486         DONE;
5487     }
5488   operands[3] = force_reg (<MODE>mode, value);
5489 })
5490
5491 (define_insn "*insv_reg<mode>"
5492   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5493                           (match_operand 1 "const_int_operand" "n")
5494                           (match_operand 2 "const_int_operand" "n"))
5495         (match_operand:GPI 3 "register_operand" "r"))]
5496   "!(UINTVAL (operands[1]) == 0
5497      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5498          > GET_MODE_BITSIZE (<MODE>mode)))"
5499   "bfi\\t%<w>0, %<w>3, %2, %1"
5500   [(set_attr "type" "bfm")]
5501 )
5502
5503 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5504   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5505                           (match_operand 1 "const_int_operand" "n")
5506                           (match_operand 2 "const_int_operand" "n"))
5507         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5508   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5509   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5510   [(set_attr "type" "bfm")]
5511 )
5512
5513 ;;  Match a bfi instruction where the shift of OP3 means that we are
5514 ;;  actually copying the least significant bits of OP3 into OP0 by way
5515 ;;  of the AND masks and the IOR instruction.  A similar instruction
5516 ;;  with the two parts of the IOR swapped around was never triggered
5517 ;;  in a bootstrap build and test of GCC so it was not included.
5518
5519 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5520   [(set (match_operand:GPI 0 "register_operand" "=r")
5521         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5522                           (match_operand:GPI 2 "const_int_operand" "n"))
5523                  (and:GPI (ashift:GPI
5524                            (match_operand:GPI 3 "register_operand" "r")
5525                            (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5526                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5527   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5528                                       UINTVAL (operands[4]),
5529                                       UINTVAL(operands[5]))"
5530   "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5531   [(set_attr "type" "bfm")]
5532 )
5533
5534 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5535   [(set (match_operand:GPI 0 "register_operand" "=r")
5536         (ior:GPI (and:GPI (ashift:GPI
5537                            (match_operand:GPI 1 "register_operand" "r")
5538                            (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5539                           (match_operand:GPI 3 "const_int_operand" "n"))
5540                  (and:GPI (match_operand:GPI 4 "register_operand" "0")
5541                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5542   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5543                                       UINTVAL (operands[2]),
5544                                       UINTVAL(operands[3]))"
5545   "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5546   [(set_attr "type" "bfm")]
5547 )
5548
5549 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5550 ;; the shift is large enough to remove the need for an AND instruction.
5551
5552 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5553   [(set (match_operand:GPI 0 "register_operand" "=r")
5554         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5555                           (match_operand:GPI 2 "const_int_operand" "n"))
5556                  (ashift:GPI
5557                           (match_operand:GPI 3 "register_operand" "r")
5558                           (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5559   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5560                                       UINTVAL (operands[4]),
5561                                       HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5562 {
5563   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5564   return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5565 }
5566   [(set_attr "type" "bfm")]
5567 )
5568
5569 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5570   [(set (match_operand:GPI 0 "register_operand" "=r")
5571         (ior:GPI (ashift:GPI
5572                           (match_operand:GPI 1 "register_operand" "r")
5573                           (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5574                  (and:GPI (match_operand:GPI 3 "register_operand" "0")
5575                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5576   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5577                                       UINTVAL (operands[2]),
5578                                       HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5579 {
5580   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5581   return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5582 }
5583   [(set_attr "type" "bfm")]
5584 )
5585
5586 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5587 ;; copying the least significant bits of OP3 to OP0.  We need two versions
5588 ;; of the instruction to handle different checks on the constant values.
5589
5590 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5591   [(set (match_operand:GPI 0 "register_operand" "=r")
5592         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5593                           (match_operand:GPI 2 "const_int_operand" "n"))
5594                  (and:GPI (match_operand:GPI 3 "register_operand" "r")
5595                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5596   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5597                                       UINTVAL (operands[4]))"
5598   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5599   [(set_attr "type" "bfm")]
5600 )
5601
5602 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5603   [(set (match_operand:GPI 0 "register_operand" "=r")
5604         (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5605                           (match_operand:GPI 4 "const_int_operand" "n"))
5606                  (and:GPI (match_operand:GPI 1 "register_operand" "0")
5607                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5608   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5609                                       UINTVAL (operands[4]))"
5610   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5611   [(set_attr "type" "bfm")]
5612 )
5613
5614 (define_insn "*extr_insv_lower_reg<mode>"
5615   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5616                           (match_operand 1 "const_int_operand" "n")
5617                           (const_int 0))
5618         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5619                           (match_dup 1)
5620                           (match_operand 3 "const_int_operand" "n")))]
5621   "!(UINTVAL (operands[1]) == 0
5622      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5623          > GET_MODE_BITSIZE (<MODE>mode)))"
5624   "bfxil\\t%<w>0, %<w>2, %3, %1"
5625   [(set_attr "type" "bfm")]
5626 )
5627
5628 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5629   [(set (match_operand:GPI 0 "register_operand" "=r")
5630         (ashift:GPI (ANY_EXTEND:GPI
5631                      (match_operand:ALLX 1 "register_operand" "r"))
5632                     (match_operand 2 "const_int_operand" "n")))]
5633   "UINTVAL (operands[2]) < <GPI:sizen>"
5634 {
5635   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5636               ? GEN_INT (<ALLX:sizen>)
5637               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5638   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5639 }
5640   [(set_attr "type" "bfx")]
5641 )
5642
5643 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5644
5645 (define_insn "*andim_ashift<mode>_bfiz"
5646   [(set (match_operand:GPI 0 "register_operand" "=r")
5647         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5648                              (match_operand 2 "const_int_operand" "n"))
5649                  (match_operand 3 "const_int_operand" "n")))]
5650   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5651   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5652   [(set_attr "type" "bfx")]
5653 )
5654
5655 ;; Match sbfiz pattern in a shift left + shift right operation.
5656
5657 (define_insn "*ashift<mode>_extv_bfiz"
5658   [(set (match_operand:GPI 0 "register_operand" "=r")
5659         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5660                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5661                                       (const_int 0))
5662                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5663   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5664              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5665   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5666   [(set_attr "type" "bfx")]
5667 )
5668
5669 ;; When the bit position and width of the equivalent extraction add up to 32
5670 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5671 ;; zero-extension of the X-reg.
5672 (define_split
5673   [(set (match_operand:DI 0 "register_operand")
5674         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5675                              (match_operand 2 "const_int_operand"))
5676                  (match_operand 3 "const_int_operand")))]
5677  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5678   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5679       == GET_MODE_BITSIZE (SImode)"
5680   [(set (match_dup 0)
5681         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5682   {
5683     operands[4] = gen_lowpart (SImode, operands[1]);
5684   }
5685 )
5686
5687 (define_insn "bswap<mode>2"
5688   [(set (match_operand:GPI 0 "register_operand" "=r")
5689         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5690   ""
5691   "rev\\t%<w>0, %<w>1"
5692   [(set_attr "type" "rev")]
5693 )
5694
5695 (define_insn "bswaphi2"
5696   [(set (match_operand:HI 0 "register_operand" "=r")
5697         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5698   ""
5699   "rev16\\t%w0, %w1"
5700   [(set_attr "type" "rev")]
5701 )
5702
5703 (define_insn "*aarch64_bfxil<mode>"
5704   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5705     (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5706                     (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5707             (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5708                     (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5709   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5710   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5711     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5712   {
5713     switch (which_alternative)
5714     {
5715       case 0:
5716         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5717         return "bfxil\\t%<w>0, %<w>1, 0, %3";
5718       case 1:
5719         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5720         return "bfxil\\t%<w>0, %<w>2, 0, %3";
5721       default:
5722         gcc_unreachable ();
5723     }
5724   }
5725   [(set_attr "type" "bfm")]
5726 )
5727
5728 ; Zero-extended version of above (aarch64_bfxil)
5729 (define_insn "*aarch64_bfxilsi_uxtw"
5730   [(set (match_operand:DI 0 "register_operand" "=r,r")
5731         (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5732                                         "r,0")
5733                     (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5734             (and:SI (match_operand:SI 2 "register_operand" "0,r")
5735                     (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5736   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5737   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5738     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5739   {
5740     switch (which_alternative)
5741     {
5742       case 0:
5743         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5744         return "bfxil\\t%0, %1, 0, %3";
5745       case 1:
5746         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5747         return "bfxil\\t%0, %2, 0, %3";
5748       default:
5749         gcc_unreachable ();
5750     }
5751   }
5752   [(set_attr "type" "bfm")]
5753 )
5754
5755 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5756 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5757 ;; each valid permutation.
5758
5759 (define_insn "rev16<mode>2"
5760   [(set (match_operand:GPI 0 "register_operand" "=r")
5761         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5762                                       (const_int 8))
5763                           (match_operand:GPI 3 "const_int_operand" "n"))
5764                  (and:GPI (lshiftrt:GPI (match_dup 1)
5765                                         (const_int 8))
5766                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5767   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5768    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5769   "rev16\\t%<w>0, %<w>1"
5770   [(set_attr "type" "rev")]
5771 )
5772
5773 (define_insn "rev16<mode>2_alt"
5774   [(set (match_operand:GPI 0 "register_operand" "=r")
5775         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5776                                         (const_int 8))
5777                           (match_operand:GPI 2 "const_int_operand" "n"))
5778                  (and:GPI (ashift:GPI (match_dup 1)
5779                                       (const_int 8))
5780                           (match_operand:GPI 3 "const_int_operand" "n"))))]
5781   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5782    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5783   "rev16\\t%<w>0, %<w>1"
5784   [(set_attr "type" "rev")]
5785 )
5786
5787 ;; zero_extend version of above
5788 (define_insn "*bswapsi2_uxtw"
5789   [(set (match_operand:DI 0 "register_operand" "=r")
5790         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5791   ""
5792   "rev\\t%w0, %w1"
5793   [(set_attr "type" "rev")]
5794 )
5795
5796 ;; -------------------------------------------------------------------
5797 ;; Floating-point intrinsics
5798 ;; -------------------------------------------------------------------
5799
5800 ;; frint floating-point round to integral standard patterns.
5801 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5802
5803 (define_insn "<frint_pattern><mode>2"
5804   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5805         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5806          FRINT))]
5807   "TARGET_FLOAT"
5808   "frint<frint_suffix>\\t%<s>0, %<s>1"
5809   [(set_attr "type" "f_rint<stype>")]
5810 )
5811
5812 ;; frcvt floating-point round to integer and convert standard patterns.
5813 ;; Expands to lbtrunc, lceil, lfloor, lround.
5814 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5815   [(set (match_operand:GPI 0 "register_operand" "=r")
5816         (FIXUORS:GPI
5817           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5818            FCVT)))]
5819   "TARGET_FLOAT"
5820   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5821   [(set_attr "type" "f_cvtf2i")]
5822 )
5823
5824 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5825   [(set (match_operand:GPI 0 "register_operand" "=r")
5826         (FIXUORS:GPI
5827           (mult:GPF
5828             (match_operand:GPF 1 "register_operand" "w")
5829             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5830   "TARGET_FLOAT
5831    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5832                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5833   {
5834     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5835     char buf[64];
5836     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5837     output_asm_insn (buf, operands);
5838     return "";
5839   }
5840   [(set_attr "type" "f_cvtf2i")]
5841 )
5842
5843 ;; fma - expand fma into patterns with the accumulator operand first since
5844 ;; reusing the accumulator results in better register allocation.
5845 ;; The register allocator considers copy preferences in operand order,
5846 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5847
5848 (define_expand "fma<mode>4"
5849   [(set (match_operand:GPF_F16 0 "register_operand")
5850         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5851                      (match_operand:GPF_F16 2 "register_operand")
5852                      (match_operand:GPF_F16 3 "register_operand")))]
5853   "TARGET_FLOAT"
5854 )
5855
5856 (define_insn "*aarch64_fma<mode>4"
5857   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5858         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5859                      (match_operand:GPF_F16 3 "register_operand" "w")
5860                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5861   "TARGET_FLOAT"
5862   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5863   [(set_attr "type" "fmac<stype>")]
5864 )
5865
5866 (define_expand "fnma<mode>4"
5867   [(set (match_operand:GPF_F16 0 "register_operand")
5868         (fma:GPF_F16
5869           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5870           (match_operand:GPF_F16 2 "register_operand")
5871           (match_operand:GPF_F16 3 "register_operand")))]
5872   "TARGET_FLOAT"
5873 )
5874
5875 (define_insn "*aarch64_fnma<mode>4"
5876   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5877         (fma:GPF_F16
5878           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5879           (match_operand:GPF_F16 3 "register_operand" "w")
5880           (match_operand:GPF_F16 1 "register_operand" "w")))]
5881   "TARGET_FLOAT"
5882   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5883   [(set_attr "type" "fmac<stype>")]
5884 )
5885
5886
5887 (define_expand "fms<mode>4"
5888   [(set (match_operand:GPF 0 "register_operand")
5889         (fma:GPF (match_operand:GPF 1 "register_operand")
5890                  (match_operand:GPF 2 "register_operand")
5891                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5892   "TARGET_FLOAT"
5893 )
5894
5895 (define_insn "*aarch64_fms<mode>4"
5896   [(set (match_operand:GPF 0 "register_operand" "=w")
5897         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5898                  (match_operand:GPF 3 "register_operand" "w")
5899                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5900   "TARGET_FLOAT"
5901   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5902   [(set_attr "type" "fmac<s>")]
5903 )
5904
5905 (define_expand "fnms<mode>4"
5906   [(set (match_operand:GPF 0 "register_operand")
5907         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5908                  (match_operand:GPF 2 "register_operand")
5909                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5910   "TARGET_FLOAT"
5911 )
5912
5913 (define_insn "*aarch64_fnms<mode>4"
5914   [(set (match_operand:GPF 0 "register_operand" "=w")
5915         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5916                  (match_operand:GPF 3 "register_operand" "w")
5917                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5918   "TARGET_FLOAT"
5919   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5920   [(set_attr "type" "fmac<s>")]
5921 )
5922
5923 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5924 (define_insn "*aarch64_fnmadd<mode>4"
5925   [(set (match_operand:GPF 0 "register_operand" "=w")
5926         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5927                           (match_operand:GPF 3 "register_operand" "w")
5928                           (match_operand:GPF 1 "register_operand" "w"))))]
5929   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5930   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5931   [(set_attr "type" "fmac<s>")]
5932 )
5933
5934 ;; -------------------------------------------------------------------
5935 ;; Floating-point conversions
5936 ;; -------------------------------------------------------------------
5937
5938 (define_insn "extendsfdf2"
5939   [(set (match_operand:DF 0 "register_operand" "=w")
5940         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5941   "TARGET_FLOAT"
5942   "fcvt\\t%d0, %s1"
5943   [(set_attr "type" "f_cvt")]
5944 )
5945
5946 (define_insn "extendhfsf2"
5947   [(set (match_operand:SF 0 "register_operand" "=w")
5948         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5949   "TARGET_FLOAT"
5950   "fcvt\\t%s0, %h1"
5951   [(set_attr "type" "f_cvt")]
5952 )
5953
5954 (define_insn "extendhfdf2"
5955   [(set (match_operand:DF 0 "register_operand" "=w")
5956         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5957   "TARGET_FLOAT"
5958   "fcvt\\t%d0, %h1"
5959   [(set_attr "type" "f_cvt")]
5960 )
5961
5962 (define_insn "truncdfsf2"
5963   [(set (match_operand:SF 0 "register_operand" "=w")
5964         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5965   "TARGET_FLOAT"
5966   "fcvt\\t%s0, %d1"
5967   [(set_attr "type" "f_cvt")]
5968 )
5969
5970 (define_insn "truncsfhf2"
5971   [(set (match_operand:HF 0 "register_operand" "=w")
5972         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5973   "TARGET_FLOAT"
5974   "fcvt\\t%h0, %s1"
5975   [(set_attr "type" "f_cvt")]
5976 )
5977
5978 (define_insn "truncdfhf2"
5979   [(set (match_operand:HF 0 "register_operand" "=w")
5980         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5981   "TARGET_FLOAT"
5982   "fcvt\\t%h0, %d1"
5983   [(set_attr "type" "f_cvt")]
5984 )
5985
5986 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5987 ;; and making r = w more expensive
5988
5989 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5990   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5991         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5992   "TARGET_FLOAT"
5993   "@
5994    fcvtz<su>\t%<s>0, %<s>1
5995    fcvtz<su>\t%<w>0, %<s>1"
5996   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5997 )
5998
5999 ;; Convert HF -> SI or DI
6000
6001 (define_insn "<optab>_trunchf<GPI:mode>2"
6002   [(set (match_operand:GPI 0 "register_operand" "=r")
6003         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6004   "TARGET_FP_F16INST"
6005   "fcvtz<su>\t%<w>0, %h1"
6006   [(set_attr "type" "f_cvtf2i")]
6007 )
6008
6009 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6010 ;; input in a fp register and output in a integer register
6011
6012 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6013   [(set (match_operand:GPI 0 "register_operand" "=r")
6014         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6015   "TARGET_FLOAT"
6016   "fcvtz<su>\t%<w>0, %<fpw>1"
6017   [(set_attr "type" "f_cvtf2i")]
6018 )
6019
6020 (define_insn "*fix_to_zero_extend<mode>di2"
6021   [(set (match_operand:DI 0 "register_operand" "=r")
6022         (zero_extend:DI
6023          (unsigned_fix:SI
6024           (match_operand:GPF 1 "register_operand" "w"))))]
6025   "TARGET_FLOAT"
6026   "fcvtzu\t%w0, %<s>1"
6027   [(set_attr "type" "f_cvtf2i")]
6028 )
6029
6030 (define_insn "<optab><fcvt_target><GPF:mode>2"
6031   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6032         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6033   "TARGET_FLOAT"
6034   "@
6035    <su_optab>cvtf\t%<GPF:s>0, %<s>1
6036    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6037   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6038    (set_attr "arch" "simd,fp")]
6039 )
6040
6041 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6042   [(set (match_operand:GPF 0 "register_operand" "=w")
6043         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6044   "TARGET_FLOAT"
6045   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6046   [(set_attr "type" "f_cvti2f")]
6047 )
6048
6049 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6050 ;; midend will arrange for an SImode conversion to HFmode to first go
6051 ;; through DFmode, then to HFmode.  But first it will try converting
6052 ;; to DImode then down, which would match our DImode pattern below and
6053 ;; give very poor code-generation.  So, we must provide our own emulation
6054 ;; of the mid-end logic.
6055
6056 (define_insn "aarch64_fp16_<optab><mode>hf2"
6057   [(set (match_operand:HF 0 "register_operand" "=w")
6058         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6059   "TARGET_FP_F16INST"
6060   "<su_optab>cvtf\t%h0, %<w>1"
6061   [(set_attr "type" "f_cvti2f")]
6062 )
6063
6064 (define_expand "<optab>sihf2"
6065   [(set (match_operand:HF 0 "register_operand")
6066         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6067   "TARGET_FLOAT"
6068 {
6069   if (TARGET_FP_F16INST)
6070     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6071   else
6072     {
6073       rtx convert_target = gen_reg_rtx (DFmode);
6074       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6075       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6076     }
6077   DONE;
6078 }
6079 )
6080
6081 ;; For DImode there is no wide enough floating-point mode that we
6082 ;; can convert through natively (TFmode would work, but requires a library
6083 ;; call).  However, we know that any value >= 65504 will be rounded
6084 ;; to infinity on conversion.  This is well within the range of SImode, so
6085 ;; we can:
6086 ;;   Saturate to SImode.
6087 ;;   Convert from that to DFmode
6088 ;;   Convert from that to HFmode (phew!).
6089 ;; Note that the saturation to SImode requires the SIMD extensions.  If
6090 ;; we ever need to provide this pattern where the SIMD extensions are not
6091 ;; available, we would need a different approach.
6092
6093 (define_expand "<optab>dihf2"
6094   [(set (match_operand:HF 0 "register_operand")
6095         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6096   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6097 {
6098   if (TARGET_FP_F16INST)
6099     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6100   else
6101     {
6102       rtx sat_target = gen_reg_rtx (SImode);
6103       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6104       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6105     }
6106
6107   DONE;
6108 }
6109 )
6110
6111 ;; Convert between fixed-point and floating-point (scalar modes)
6112
6113 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6114   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6115         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6116                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6117          FCVT_F2FIXED))]
6118   ""
6119   "@
6120    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6121    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6122   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6123    (set_attr "arch" "fp,simd")]
6124 )
6125
6126 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6127   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6128         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6129                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6130          FCVT_FIXED2F))]
6131   ""
6132   "@
6133    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6134    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6135   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6136    (set_attr "arch" "fp,simd")]
6137 )
6138
6139 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6140   [(set (match_operand:GPI 0 "register_operand" "=r")
6141         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6142                      (match_operand:SI 2 "immediate_operand" "i")]
6143          FCVT_F2FIXED))]
6144   "TARGET_FP_F16INST"
6145    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6146   [(set_attr "type" "f_cvtf2i")]
6147 )
6148
6149 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6150   [(set (match_operand:HF 0 "register_operand" "=w")
6151         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6152                     (match_operand:SI 2 "immediate_operand" "i")]
6153          FCVT_FIXED2F))]
6154   "TARGET_FP_F16INST"
6155   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6156   [(set_attr "type" "f_cvti2f")]
6157 )
6158
6159 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6160   [(set (match_operand:HI 0 "register_operand" "=w")
6161         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6162                     (match_operand:SI 2 "immediate_operand" "i")]
6163          FCVT_F2FIXED))]
6164   "TARGET_SIMD"
6165   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6166   [(set_attr "type" "neon_fp_to_int_s")]
6167 )
6168
6169 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6170   [(set (match_operand:HF 0 "register_operand" "=w")
6171         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6172                     (match_operand:SI 2 "immediate_operand" "i")]
6173          FCVT_FIXED2F))]
6174   "TARGET_SIMD"
6175   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6176   [(set_attr "type" "neon_int_to_fp_s")]
6177 )
6178
6179 ;; -------------------------------------------------------------------
6180 ;; Floating-point arithmetic
6181 ;; -------------------------------------------------------------------
6182
6183 (define_insn "add<mode>3"
6184   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6185         (plus:GPF_F16
6186          (match_operand:GPF_F16 1 "register_operand" "w")
6187          (match_operand:GPF_F16 2 "register_operand" "w")))]
6188   "TARGET_FLOAT"
6189   "fadd\\t%<s>0, %<s>1, %<s>2"
6190   [(set_attr "type" "fadd<stype>")]
6191 )
6192
6193 (define_insn "sub<mode>3"
6194   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6195         (minus:GPF_F16
6196          (match_operand:GPF_F16 1 "register_operand" "w")
6197          (match_operand:GPF_F16 2 "register_operand" "w")))]
6198   "TARGET_FLOAT"
6199   "fsub\\t%<s>0, %<s>1, %<s>2"
6200   [(set_attr "type" "fadd<stype>")]
6201 )
6202
6203 (define_insn "mul<mode>3"
6204   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6205         (mult:GPF_F16
6206          (match_operand:GPF_F16 1 "register_operand" "w")
6207          (match_operand:GPF_F16 2 "register_operand" "w")))]
6208   "TARGET_FLOAT"
6209   "fmul\\t%<s>0, %<s>1, %<s>2"
6210   [(set_attr "type" "fmul<stype>")]
6211 )
6212
6213 (define_insn "*fnmul<mode>3"
6214   [(set (match_operand:GPF 0 "register_operand" "=w")
6215         (mult:GPF
6216                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6217                  (match_operand:GPF 2 "register_operand" "w")))]
6218   "TARGET_FLOAT && !flag_rounding_math"
6219   "fnmul\\t%<s>0, %<s>1, %<s>2"
6220   [(set_attr "type" "fmul<s>")]
6221 )
6222
6223 (define_insn "*fnmul<mode>3"
6224   [(set (match_operand:GPF 0 "register_operand" "=w")
6225         (neg:GPF (mult:GPF
6226                  (match_operand:GPF 1 "register_operand" "w")
6227                  (match_operand:GPF 2 "register_operand" "w"))))]
6228   "TARGET_FLOAT"
6229   "fnmul\\t%<s>0, %<s>1, %<s>2"
6230   [(set_attr "type" "fmul<s>")]
6231 )
6232
6233 (define_expand "div<mode>3"
6234  [(set (match_operand:GPF_F16 0 "register_operand")
6235        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6236                     (match_operand:GPF_F16 2 "register_operand")))]
6237  "TARGET_FLOAT"
6238 {
6239   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6240     DONE;
6241
6242   operands[1] = force_reg (<MODE>mode, operands[1]);
6243 })
6244
6245 (define_insn "*div<mode>3"
6246   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6247         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6248                      (match_operand:GPF_F16 2 "register_operand" "w")))]
6249   "TARGET_FLOAT"
6250   "fdiv\\t%<s>0, %<s>1, %<s>2"
6251   [(set_attr "type" "fdiv<stype>")]
6252 )
6253
6254 (define_insn "neg<mode>2"
6255   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6256         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6257   "TARGET_FLOAT"
6258   "fneg\\t%<s>0, %<s>1"
6259   [(set_attr "type" "ffarith<stype>")]
6260 )
6261
6262 (define_expand "sqrt<mode>2"
6263   [(set (match_operand:GPF_F16 0 "register_operand")
6264         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6265   "TARGET_FLOAT"
6266 {
6267   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6268     DONE;
6269 })
6270
6271 (define_insn "*sqrt<mode>2"
6272   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6273         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6274   "TARGET_FLOAT"
6275   "fsqrt\\t%<s>0, %<s>1"
6276   [(set_attr "type" "fsqrt<stype>")]
6277 )
6278
6279 (define_insn "abs<mode>2"
6280   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6281         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6282   "TARGET_FLOAT"
6283   "fabs\\t%<s>0, %<s>1"
6284   [(set_attr "type" "ffarith<stype>")]
6285 )
6286
6287 ;; Given that smax/smin do not specify the result when either input is NaN,
6288 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6289 ;; for smin.
6290
6291 (define_insn "smax<mode>3"
6292   [(set (match_operand:GPF 0 "register_operand" "=w")
6293         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6294                   (match_operand:GPF 2 "register_operand" "w")))]
6295   "TARGET_FLOAT"
6296   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6297   [(set_attr "type" "f_minmax<s>")]
6298 )
6299
6300 (define_insn "smin<mode>3"
6301   [(set (match_operand:GPF 0 "register_operand" "=w")
6302         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6303                   (match_operand:GPF 2 "register_operand" "w")))]
6304   "TARGET_FLOAT"
6305   "fminnm\\t%<s>0, %<s>1, %<s>2"
6306   [(set_attr "type" "f_minmax<s>")]
6307 )
6308
6309 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6310 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6311 ;; which implement the IEEE fmax ()/fmin () functions.
6312 (define_insn "<maxmin_uns><mode>3"
6313   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6314         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6315                      (match_operand:GPF_F16 2 "register_operand" "w")]
6316                      FMAXMIN_UNS))]
6317   "TARGET_FLOAT"
6318   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6319   [(set_attr "type" "f_minmax<stype>")]
6320 )
6321
6322 (define_expand "lrint<GPF:mode><GPI:mode>2"
6323   [(match_operand:GPI 0 "register_operand")
6324    (match_operand:GPF 1 "register_operand")]
6325   "TARGET_FLOAT
6326    && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
6327    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6328 {
6329   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6330   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6331   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6332   DONE;
6333 }
6334 )
6335
6336 ;; For copysign (x, y), we want to generate:
6337 ;;
6338 ;;   LDR d2, #(1 << 63)
6339 ;;   BSL v2.8b, [y], [x]
6340 ;;
6341 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.  Because
6342 ;; we expect these operations to nearly always operate on
6343 ;; floating-point values, we do not want the operation to be
6344 ;; simplified into a bit-field insert operation that operates on the
6345 ;; integer side, since typically that would involve three inter-bank
6346 ;; register copies.  As we do not expect copysign to be followed by
6347 ;; other logical operations on the result, it seems preferable to keep
6348 ;; this as an unspec operation, rather than exposing the underlying
6349 ;; logic to the compiler.
6350
6351 (define_expand "copysign<GPF:mode>3"
6352   [(match_operand:GPF 0 "register_operand")
6353    (match_operand:GPF 1 "register_operand")
6354    (match_operand:GPF 2 "register_operand")]
6355   "TARGET_FLOAT && TARGET_SIMD"
6356 {
6357   rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6358   emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6359                                     << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6360   emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6361                                        bitmask));
6362   DONE;
6363 }
6364 )
6365
6366 (define_insn "copysign<GPF:mode>3_insn"
6367   [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6368         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6369                      (match_operand:GPF 2 "register_operand" "w,w,0,0")
6370                      (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6371          UNSPEC_COPYSIGN))]
6372   "TARGET_FLOAT && TARGET_SIMD"
6373   "@
6374    bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6375    bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6376    bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6377    bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6378   [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6379 )
6380
6381
6382 ;; For xorsign (x, y), we want to generate:
6383 ;;
6384 ;; LDR   d2, #1<<63
6385 ;; AND   v3.8B, v1.8B, v2.8B
6386 ;; EOR   v0.8B, v0.8B, v3.8B
6387 ;;
6388
6389 (define_expand "xorsign<mode>3"
6390   [(match_operand:GPF 0 "register_operand")
6391    (match_operand:GPF 1 "register_operand")
6392    (match_operand:GPF 2 "register_operand")]
6393   "TARGET_FLOAT && TARGET_SIMD"
6394 {
6395
6396   machine_mode imode = <V_INT_EQUIV>mode;
6397   rtx mask = gen_reg_rtx (imode);
6398   rtx op1x = gen_reg_rtx (imode);
6399   rtx op2x = gen_reg_rtx (imode);
6400
6401   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6402   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6403                                                      imode)));
6404
6405   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6406                                     lowpart_subreg (imode, operands[2],
6407                                                     <MODE>mode)));
6408   emit_insn (gen_xor<v_int_equiv>3 (op1x,
6409                                     lowpart_subreg (imode, operands[1],
6410                                                     <MODE>mode),
6411                                     op2x));
6412   emit_move_insn (operands[0],
6413                   lowpart_subreg (<MODE>mode, op1x, imode));
6414   DONE;
6415 }
6416 )
6417
6418 ;; -------------------------------------------------------------------
6419 ;; Reload support
6420 ;; -------------------------------------------------------------------
6421 ;; Reload Scalar Floating point modes from constant pool.
6422 ;; The AArch64 port doesn't have __int128 constant move support.
6423 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6424 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6425  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6426        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6427   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6428  "TARGET_FLOAT"
6429  {
6430    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6431    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6432    DONE;
6433  }
6434 )
6435
6436 ;; Reload Vector modes from constant pool.
6437 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6438  [(set (match_operand:VALL 0 "register_operand" "=w")
6439        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6440   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6441  "TARGET_FLOAT"
6442  {
6443    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6444    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6445    DONE;
6446  }
6447 )
6448
6449 (define_expand "@aarch64_reload_mov<mode>"
6450   [(set (match_operand:TX 0 "register_operand" "=w")
6451         (match_operand:TX 1 "register_operand" "w"))
6452    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6453   ]
6454   "TARGET_FLOAT"
6455   {
6456     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6457     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6458     gen_aarch64_movtilow_tilow (op0, op1);
6459     gen_aarch64_movdi_tihigh (operands[2], op1);
6460     gen_aarch64_movtihigh_di (op0, operands[2]);
6461     DONE;
6462   }
6463 )
6464
6465 ;; The following secondary reload helpers patterns are invoked
6466 ;; after or during reload as we don't want these patterns to start
6467 ;; kicking in during the combiner.
6468
6469 (define_insn "@aarch64_movdi_<mode>low"
6470   [(set (match_operand:DI 0 "register_operand" "=r")
6471         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6472                          (const_int 64) (const_int 0)))]
6473   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6474   "fmov\\t%x0, %d1"
6475   [(set_attr "type" "f_mrc")
6476    (set_attr "length" "4")
6477   ])
6478
6479 (define_insn "@aarch64_movdi_<mode>high"
6480   [(set (match_operand:DI 0 "register_operand" "=r")
6481         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6482                          (const_int 64) (const_int 64)))]
6483   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6484   "fmov\\t%x0, %1.d[1]"
6485   [(set_attr "type" "f_mrc")
6486    (set_attr "length" "4")
6487   ])
6488
6489 (define_insn "@aarch64_mov<mode>high_di"
6490   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6491                          (const_int 64) (const_int 64))
6492         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6493   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6494   "fmov\\t%0.d[1], %x1"
6495   [(set_attr "type" "f_mcr")
6496    (set_attr "length" "4")
6497   ])
6498
6499 (define_insn "@aarch64_mov<mode>low_di"
6500   [(set (match_operand:TX 0 "register_operand" "=w")
6501         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6502   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6503   "fmov\\t%d0, %x1"
6504   [(set_attr "type" "f_mcr")
6505    (set_attr "length" "4")
6506   ])
6507
6508 (define_insn "aarch64_movtilow_tilow"
6509   [(set (match_operand:TI 0 "register_operand" "=w")
6510         (zero_extend:TI
6511           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6512   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6513   "fmov\\t%d0, %d1"
6514   [(set_attr "type" "fmov")
6515    (set_attr "length" "4")
6516   ])
6517
6518 ;; There is a deliberate reason why the parameters of high and lo_sum's
6519 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6520 ;; and lo_sum's to be used with the labels defining the jump tables in
6521 ;; rodata section.
6522
6523 (define_expand "add_losym"
6524   [(set (match_operand 0 "register_operand")
6525         (lo_sum (match_operand 1 "register_operand")
6526                 (match_operand 2 "aarch64_valid_symref")))]
6527   ""
6528 {
6529   machine_mode mode = GET_MODE (operands[0]);
6530
6531   emit_insn ((mode == DImode
6532               ? gen_add_losym_di
6533               : gen_add_losym_si) (operands[0],
6534                                    operands[1],
6535                                    operands[2]));
6536   DONE;
6537 })
6538
6539 (define_insn "add_losym_<mode>"
6540   [(set (match_operand:P 0 "register_operand" "=r")
6541         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6542                   (match_operand 2 "aarch64_valid_symref" "S")))]
6543   ""
6544   "add\\t%<w>0, %<w>1, :lo12:%c2"
6545   [(set_attr "type" "alu_imm")]
6546 )
6547
6548 (define_insn "ldr_got_small_<mode>"
6549   [(set (match_operand:PTR 0 "register_operand" "=r")
6550         (unspec:PTR [(mem:PTR (lo_sum:PTR
6551                               (match_operand:PTR 1 "register_operand" "r")
6552                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6553                     UNSPEC_GOTSMALLPIC))]
6554   ""
6555   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6556   [(set_attr "type" "load_<ldst_sz>")]
6557 )
6558
6559 (define_insn "ldr_got_small_sidi"
6560   [(set (match_operand:DI 0 "register_operand" "=r")
6561         (zero_extend:DI
6562          (unspec:SI [(mem:SI (lo_sum:DI
6563                              (match_operand:DI 1 "register_operand" "r")
6564                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6565                     UNSPEC_GOTSMALLPIC)))]
6566   "TARGET_ILP32"
6567   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6568   [(set_attr "type" "load_4")]
6569 )
6570
6571 (define_insn "ldr_got_small_28k_<mode>"
6572   [(set (match_operand:PTR 0 "register_operand" "=r")
6573         (unspec:PTR [(mem:PTR (lo_sum:PTR
6574                               (match_operand:PTR 1 "register_operand" "r")
6575                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6576                     UNSPEC_GOTSMALLPIC28K))]
6577   ""
6578   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6579   [(set_attr "type" "load_<ldst_sz>")]
6580 )
6581
6582 (define_insn "ldr_got_small_28k_sidi"
6583   [(set (match_operand:DI 0 "register_operand" "=r")
6584         (zero_extend:DI
6585          (unspec:SI [(mem:SI (lo_sum:DI
6586                              (match_operand:DI 1 "register_operand" "r")
6587                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6588                     UNSPEC_GOTSMALLPIC28K)))]
6589   "TARGET_ILP32"
6590   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6591   [(set_attr "type" "load_4")]
6592 )
6593
6594 (define_insn "ldr_got_tiny"
6595   [(set (match_operand:DI 0 "register_operand" "=r")
6596         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6597                    UNSPEC_GOTTINYPIC))]
6598   ""
6599   "ldr\\t%0, %L1"
6600   [(set_attr "type" "load_8")]
6601 )
6602
6603 (define_insn "aarch64_load_tp_hard"
6604   [(set (match_operand:DI 0 "register_operand" "=r")
6605         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6606   ""
6607   "mrs\\t%0, tpidr_el0"
6608   [(set_attr "type" "mrs")]
6609 )
6610
6611 ;; The TLS ABI specifically requires that the compiler does not schedule
6612 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6613 ;; Therefore we treat the stubs as an atomic sequence.
6614 (define_expand "tlsgd_small_<mode>"
6615  [(parallel [(set (match_operand 0 "register_operand")
6616                   (call (mem:DI (match_dup 2)) (const_int 1)))
6617              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6618              (clobber (reg:DI LR_REGNUM))])]
6619  ""
6620 {
6621   operands[2] = aarch64_tls_get_addr ();
6622 })
6623
6624 (define_insn "*tlsgd_small_<mode>"
6625   [(set (match_operand 0 "register_operand" "")
6626         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6627    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6628    (clobber (reg:DI LR_REGNUM))
6629   ]
6630   ""
6631   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6632   [(set_attr "type" "call")
6633    (set_attr "length" "16")])
6634
6635 (define_insn "tlsie_small_<mode>"
6636   [(set (match_operand:PTR 0 "register_operand" "=r")
6637         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6638                    UNSPEC_GOTSMALLTLS))]
6639   ""
6640   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6641   [(set_attr "type" "load_4")
6642    (set_attr "length" "8")]
6643 )
6644
6645 (define_insn "tlsie_small_sidi"
6646   [(set (match_operand:DI 0 "register_operand" "=r")
6647         (zero_extend:DI
6648           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6649                       UNSPEC_GOTSMALLTLS)))]
6650   ""
6651   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6652   [(set_attr "type" "load_4")
6653    (set_attr "length" "8")]
6654 )
6655
6656 (define_insn "tlsie_tiny_<mode>"
6657   [(set (match_operand:PTR 0 "register_operand" "=&r")
6658         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6659                      (match_operand:PTR 2 "register_operand" "r")]
6660                    UNSPEC_GOTTINYTLS))]
6661   ""
6662   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6663   [(set_attr "type" "multiple")
6664    (set_attr "length" "8")]
6665 )
6666
6667 (define_insn "tlsie_tiny_sidi"
6668   [(set (match_operand:DI 0 "register_operand" "=&r")
6669         (zero_extend:DI
6670           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6671                       (match_operand:DI 2 "register_operand" "r")
6672                       ]
6673                       UNSPEC_GOTTINYTLS)))]
6674   ""
6675   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6676   [(set_attr "type" "multiple")
6677    (set_attr "length" "8")]
6678 )
6679
6680 (define_insn "tlsle12_<mode>"
6681   [(set (match_operand:P 0 "register_operand" "=r")
6682         (unspec:P [(match_operand:P 1 "register_operand" "r")
6683                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6684                    UNSPEC_TLSLE12))]
6685   ""
6686   "add\\t%<w>0, %<w>1, #%L2";
6687   [(set_attr "type" "alu_sreg")
6688    (set_attr "length" "4")]
6689 )
6690
6691 (define_insn "tlsle24_<mode>"
6692   [(set (match_operand:P 0 "register_operand" "=r")
6693         (unspec:P [(match_operand:P 1 "register_operand" "r")
6694                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6695                    UNSPEC_TLSLE24))]
6696   ""
6697   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6698   [(set_attr "type" "multiple")
6699    (set_attr "length" "8")]
6700 )
6701
6702 (define_insn "tlsle32_<mode>"
6703   [(set (match_operand:P 0 "register_operand" "=r")
6704         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6705                    UNSPEC_TLSLE32))]
6706   ""
6707   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6708   [(set_attr "type" "multiple")
6709    (set_attr "length" "8")]
6710 )
6711
6712 (define_insn "tlsle48_<mode>"
6713   [(set (match_operand:P 0 "register_operand" "=r")
6714         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6715                    UNSPEC_TLSLE48))]
6716   ""
6717   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6718   [(set_attr "type" "multiple")
6719    (set_attr "length" "12")]
6720 )
6721
6722 (define_expand "tlsdesc_small_<mode>"
6723   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6724   "TARGET_TLS_DESC"
6725   {
6726     if (TARGET_SVE)
6727       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6728     else
6729       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6730     DONE;
6731   }
6732 )
6733
6734 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6735 ;; R0 and LR.
6736 (define_insn "tlsdesc_small_advsimd_<mode>"
6737   [(set (reg:PTR R0_REGNUM)
6738         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6739                     UNSPEC_TLSDESC))
6740    (clobber (reg:DI LR_REGNUM))
6741    (clobber (reg:CC CC_REGNUM))
6742    (clobber (match_scratch:DI 1 "=r"))
6743    (use (reg:DI FP_REGNUM))]
6744   "TARGET_TLS_DESC && !TARGET_SVE"
6745   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6746   [(set_attr "type" "call")
6747    (set_attr "length" "16")])
6748
6749 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6750 ;; all vector registers, and clobber all predicate registers, on
6751 ;; top of the usual R0 and LR.
6752 (define_insn "tlsdesc_small_sve_<mode>"
6753   [(set (reg:PTR R0_REGNUM)
6754         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6755                     UNSPEC_TLSDESC))
6756    (clobber (reg:DI LR_REGNUM))
6757    (clobber (reg:CC CC_REGNUM))
6758    (clobber_high (reg:TI V0_REGNUM))
6759    (clobber_high (reg:TI V1_REGNUM))
6760    (clobber_high (reg:TI V2_REGNUM))
6761    (clobber_high (reg:TI V3_REGNUM))
6762    (clobber_high (reg:TI V4_REGNUM))
6763    (clobber_high (reg:TI V5_REGNUM))
6764    (clobber_high (reg:TI V6_REGNUM))
6765    (clobber_high (reg:TI V7_REGNUM))
6766    (clobber_high (reg:TI V8_REGNUM))
6767    (clobber_high (reg:TI V9_REGNUM))
6768    (clobber_high (reg:TI V10_REGNUM))
6769    (clobber_high (reg:TI V11_REGNUM))
6770    (clobber_high (reg:TI V12_REGNUM))
6771    (clobber_high (reg:TI V13_REGNUM))
6772    (clobber_high (reg:TI V14_REGNUM))
6773    (clobber_high (reg:TI V15_REGNUM))
6774    (clobber_high (reg:TI V16_REGNUM))
6775    (clobber_high (reg:TI V17_REGNUM))
6776    (clobber_high (reg:TI V18_REGNUM))
6777    (clobber_high (reg:TI V19_REGNUM))
6778    (clobber_high (reg:TI V20_REGNUM))
6779    (clobber_high (reg:TI V21_REGNUM))
6780    (clobber_high (reg:TI V22_REGNUM))
6781    (clobber_high (reg:TI V23_REGNUM))
6782    (clobber_high (reg:TI V24_REGNUM))
6783    (clobber_high (reg:TI V25_REGNUM))
6784    (clobber_high (reg:TI V26_REGNUM))
6785    (clobber_high (reg:TI V27_REGNUM))
6786    (clobber_high (reg:TI V28_REGNUM))
6787    (clobber_high (reg:TI V29_REGNUM))
6788    (clobber_high (reg:TI V30_REGNUM))
6789    (clobber_high (reg:TI V31_REGNUM))
6790    (clobber (reg:VNx2BI P0_REGNUM))
6791    (clobber (reg:VNx2BI P1_REGNUM))
6792    (clobber (reg:VNx2BI P2_REGNUM))
6793    (clobber (reg:VNx2BI P3_REGNUM))
6794    (clobber (reg:VNx2BI P4_REGNUM))
6795    (clobber (reg:VNx2BI P5_REGNUM))
6796    (clobber (reg:VNx2BI P6_REGNUM))
6797    (clobber (reg:VNx2BI P7_REGNUM))
6798    (clobber (reg:VNx2BI P8_REGNUM))
6799    (clobber (reg:VNx2BI P9_REGNUM))
6800    (clobber (reg:VNx2BI P10_REGNUM))
6801    (clobber (reg:VNx2BI P11_REGNUM))
6802    (clobber (reg:VNx2BI P12_REGNUM))
6803    (clobber (reg:VNx2BI P13_REGNUM))
6804    (clobber (reg:VNx2BI P14_REGNUM))
6805    (clobber (reg:VNx2BI P15_REGNUM))
6806    (clobber (match_scratch:DI 1 "=r"))
6807    (use (reg:DI FP_REGNUM))]
6808   "TARGET_TLS_DESC && TARGET_SVE"
6809   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6810   [(set_attr "type" "call")
6811    (set_attr "length" "16")])
6812
6813 (define_insn "stack_tie"
6814   [(set (mem:BLK (scratch))
6815         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6816                      (match_operand:DI 1 "register_operand" "rk")]
6817                     UNSPEC_PRLG_STK))]
6818   ""
6819   ""
6820   [(set_attr "length" "0")]
6821 )
6822
6823 ;; Pointer authentication patterns are always provided.  In architecture
6824 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6825 ;; This lets the user write portable software which authenticates pointers
6826 ;; when run on something which implements ARMv8.3-A, and which runs
6827 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6828 ;; implemented.
6829
6830 ;; Signing/Authenticating R30 using SP as the salt.
6831
6832 (define_insn "<pauth_mnem_prefix>sp"
6833   [(set (reg:DI R30_REGNUM)
6834         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6835   ""
6836   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6837 )
6838
6839 ;; Signing/Authenticating X17 using X16 as the salt.
6840
6841 (define_insn "<pauth_mnem_prefix>1716"
6842   [(set (reg:DI R17_REGNUM)
6843         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6844   ""
6845   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6846 )
6847
6848 ;; Stripping the signature in R30.
6849
6850 (define_insn "xpaclri"
6851   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6852   ""
6853   "hint\t7 // xpaclri"
6854 )
6855
6856 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6857 ;; all of memory.  This blocks insns from being moved across this point.
6858
6859 (define_insn "blockage"
6860   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6861   ""
6862   ""
6863   [(set_attr "length" "0")
6864    (set_attr "type" "block")]
6865 )
6866
6867 (define_insn "probe_stack_range"
6868   [(set (match_operand:DI 0 "register_operand" "=rk")
6869         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6870                              (match_operand:DI 2 "register_operand" "r")]
6871                               UNSPECV_PROBE_STACK_RANGE))]
6872   ""
6873 {
6874   return aarch64_output_probe_stack_range (operands[0], operands[2]);
6875 }
6876   [(set_attr "length" "32")]
6877 )
6878
6879 ;; This instruction is used to generate the stack clash stack adjustment and
6880 ;; probing loop.  We can't change the control flow during prologue and epilogue
6881 ;; code generation.  So we must emit a volatile unspec and expand it later on.
6882
6883 (define_insn "@probe_sve_stack_clash_<mode>"
6884   [(set (match_operand:P 0 "register_operand" "=rk")
6885         (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6886                             (match_operand:P 2 "register_operand" "r")
6887                             (match_operand:P 3 "const_int_operand" "n")
6888                             (match_operand:P 4 "aarch64_plus_immediate" "L")]
6889                              UNSPECV_PROBE_STACK_RANGE))]
6890   "TARGET_SVE"
6891 {
6892   return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6893                                                operands[3], operands[4]);
6894 }
6895   [(set_attr "length" "28")]
6896 )
6897
6898 ;; Named pattern for expanding thread pointer reference.
6899 (define_expand "get_thread_pointerdi"
6900   [(match_operand:DI 0 "register_operand")]
6901   ""
6902 {
6903   rtx tmp = aarch64_load_tp (operands[0]);
6904   if (tmp != operands[0])
6905     emit_move_insn (operands[0], tmp);
6906   DONE;
6907 })
6908
6909 ;; Named patterns for stack smashing protection.
6910 (define_expand "stack_protect_set"
6911   [(match_operand 0 "memory_operand")
6912    (match_operand 1 "memory_operand")]
6913   ""
6914 {
6915   machine_mode mode = GET_MODE (operands[0]);
6916   if (aarch64_stack_protector_guard != SSP_GLOBAL)
6917   {
6918     /* Generate access through the system register.  */
6919     rtx tmp_reg = gen_reg_rtx (mode);
6920     if (mode == DImode)
6921     {
6922         emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6923         emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6924                                GEN_INT (aarch64_stack_protector_guard_offset)));
6925     }
6926     else
6927     {
6928         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
6929         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
6930                                GEN_INT (aarch64_stack_protector_guard_offset)));
6931
6932     }
6933     operands[1] = gen_rtx_MEM (mode, tmp_reg);
6934   }
6935
6936   emit_insn ((mode == DImode
6937               ? gen_stack_protect_set_di
6938               : gen_stack_protect_set_si) (operands[0], operands[1]));
6939   DONE;
6940 })
6941
6942 (define_insn "reg_stack_protect_address_<mode>"
6943  [(set (match_operand:PTR 0 "register_operand" "=r")
6944        (unspec:PTR [(const_int 0)]
6945         UNSPEC_SSP_SYSREG))]
6946  "aarch64_stack_protector_guard != SSP_GLOBAL"
6947  {
6948    char buf[150];
6949    snprintf (buf, 150, "mrs\\t%%<w>0, %s",
6950             aarch64_stack_protector_guard_reg_str);
6951    output_asm_insn (buf, operands);
6952    return "";
6953  }
6954  [(set_attr "type" "mrs")])
6955
6956 (define_insn "stack_protect_set_<mode>"
6957   [(set (match_operand:PTR 0 "memory_operand" "=m")
6958         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6959          UNSPEC_SP_SET))
6960    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6961   ""
6962   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
6963   [(set_attr "length" "12")
6964    (set_attr "type" "multiple")])
6965
6966 (define_expand "stack_protect_test"
6967   [(match_operand 0 "memory_operand")
6968    (match_operand 1 "memory_operand")
6969    (match_operand 2)]
6970   ""
6971 {
6972   rtx result;
6973   machine_mode mode = GET_MODE (operands[0]);
6974
6975   result = gen_reg_rtx(mode);
6976   if (aarch64_stack_protector_guard != SSP_GLOBAL)
6977   {
6978     /* Generate access through the system register. The
6979        sequence we want here is the access
6980        of the stack offset to come with
6981        mrs scratch_reg, <system_register>
6982        add scratch_reg, scratch_reg, :lo12:offset. */
6983     rtx tmp_reg = gen_reg_rtx (mode);
6984     if (mode == DImode)
6985     {
6986        emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6987        emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6988                               GEN_INT (aarch64_stack_protector_guard_offset)));
6989     }
6990     else
6991     {
6992         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
6993         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
6994                                GEN_INT (aarch64_stack_protector_guard_offset)));
6995
6996     }
6997     operands[1] = gen_rtx_MEM (mode, tmp_reg);
6998   }
6999   emit_insn ((mode == DImode
7000                   ? gen_stack_protect_test_di
7001                   : gen_stack_protect_test_si) (result,
7002                                                 operands[0],
7003                                                 operands[1]));
7004
7005   if (mode == DImode)
7006     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7007                                     result, const0_rtx, operands[2]));
7008   else
7009     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7010                                     result, const0_rtx, operands[2]));
7011   DONE;
7012 })
7013
7014 (define_insn "stack_protect_test_<mode>"
7015   [(set (match_operand:PTR 0 "register_operand" "=r")
7016         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7017                      (match_operand:PTR 2 "memory_operand" "m")]
7018          UNSPEC_SP_TEST))
7019    (clobber (match_scratch:PTR 3 "=&r"))]
7020   ""
7021   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7022   [(set_attr "length" "12")
7023    (set_attr "type" "multiple")])
7024
7025 ;; Write Floating-point Control Register.
7026 (define_insn "set_fpcr"
7027   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7028   ""
7029   "msr\\tfpcr, %0"
7030   [(set_attr "type" "mrs")])
7031
7032 ;; Read Floating-point Control Register.
7033 (define_insn "get_fpcr"
7034   [(set (match_operand:SI 0 "register_operand" "=r")
7035         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7036   ""
7037   "mrs\\t%0, fpcr"
7038   [(set_attr "type" "mrs")])
7039
7040 ;; Write Floating-point Status Register.
7041 (define_insn "set_fpsr"
7042   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7043   ""
7044   "msr\\tfpsr, %0"
7045   [(set_attr "type" "mrs")])
7046
7047 ;; Read Floating-point Status Register.
7048 (define_insn "get_fpsr"
7049   [(set (match_operand:SI 0 "register_operand" "=r")
7050         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7051   ""
7052   "mrs\\t%0, fpsr"
7053   [(set_attr "type" "mrs")])
7054
7055
7056 ;; Define the subtract-one-and-jump insns so loop.c
7057 ;; knows what to generate.
7058 (define_expand "doloop_end"
7059   [(use (match_operand 0 "" ""))      ; loop pseudo
7060    (use (match_operand 1 "" ""))]     ; label
7061   "optimize > 0 && flag_modulo_sched"
7062 {
7063   rtx s0;
7064   rtx bcomp;
7065   rtx loc_ref;
7066   rtx cc_reg;
7067   rtx insn;
7068   rtx cmp;
7069
7070   /* Currently SMS relies on the do-loop pattern to recognize loops
7071      where (1) the control part consists of all insns defining and/or
7072      using a certain 'count' register and (2) the loop count can be
7073      adjusted by modifying this register prior to the loop.
7074      ??? The possible introduction of a new block to initialize the
7075      new IV can potentially affect branch optimizations.  */
7076
7077   if (GET_MODE (operands[0]) != DImode)
7078     FAIL;
7079
7080   s0 = operands [0];
7081   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7082
7083   cmp = XVECEXP (PATTERN (insn), 0, 0);
7084   cc_reg = SET_DEST (cmp);
7085   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7086   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7087   emit_jump_insn (gen_rtx_SET (pc_rtx,
7088                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7089                                                      loc_ref, pc_rtx)));
7090   DONE;
7091 })
7092
7093 ;; Track speculation through conditional branches.  We assume that
7094 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7095 (define_insn "speculation_tracker"
7096   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7097         (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7098          UNSPEC_SPECULATION_TRACKER))]
7099   ""
7100   {
7101     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7102     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7103     return "";
7104   }
7105   [(set_attr "type" "csel")]
7106 )
7107
7108 ;; BTI <target> instructions
7109 (define_insn "bti_noarg"
7110   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7111   ""
7112   "hint\t32 // bti"
7113   [(set_attr "type" "no_insn")]
7114 )
7115
7116 (define_insn "bti_c"
7117   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7118   ""
7119   "hint\t34 // bti c"
7120   [(set_attr "type" "no_insn")]
7121 )
7122
7123 (define_insn "bti_j"
7124   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7125   ""
7126   "hint\t36 // bti j"
7127   [(set_attr "type" "no_insn")]
7128 )
7129
7130 (define_insn "bti_jc"
7131   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7132   ""
7133   "hint\t38 // bti jc"
7134   [(set_attr "type" "no_insn")]
7135 )
7136
7137 ;; Helper for aarch64.c code.
7138 (define_expand "set_clobber_cc_nzc"
7139   [(parallel [(set (match_operand 0)
7140                    (match_operand 1))
7141               (clobber (reg:CC_NZC CC_REGNUM))])])
7142
7143 ;; Hard speculation barrier.
7144 (define_insn "speculation_barrier"
7145   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7146   ""
7147   "isb\;dsb\\tsy"
7148   [(set_attr "length" "8")
7149    (set_attr "type" "block")
7150    (set_attr "speculation_barrier" "true")]
7151 )
7152
7153 ;; Support for __builtin_speculation_safe_value when we have speculation
7154 ;; tracking enabled.  Use the speculation tracker to decide whether to
7155 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7156 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7157   [(set (match_operand:ALLI_TI 0 "register_operand")
7158         (unspec_volatile:ALLI_TI
7159          [(match_operand:ALLI_TI 1 "register_operand")
7160           (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7161           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7162           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7163   ""
7164   "
7165   {
7166     if (operands[2] == const0_rtx)
7167       {
7168         rtx tracker;
7169         if (<MODE>mode == TImode)
7170           tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7171         else
7172           tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7173
7174         emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7175                                                  tracker));
7176         DONE;
7177       }
7178   }
7179   "
7180 )
7181
7182 ;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
7183 ;; encoding for CSDB, but will work in older versions of the assembler.
7184 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7185   [(set (match_operand:ALLI 0 "register_operand" "=r")
7186         (unspec_volatile:ALLI
7187          [(match_operand:ALLI 1 "register_operand" "r")
7188           (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7189           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7190           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7191   ""
7192   {
7193     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7194     output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7195                      operands);
7196     return "";
7197   }
7198   [(set_attr "length" "12")
7199    (set_attr "type" "block")
7200    (set_attr "speculation_barrier" "true")]
7201 )
7202
7203 ;; Pattern to match despeculate_copyti
7204 (define_insn "*despeculate_copyti_insn"
7205   [(set (match_operand:TI 0 "register_operand" "=r")
7206         (unspec_volatile:TI
7207          [(match_operand:TI 1 "register_operand" "r")
7208           (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7209           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7210           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7211   ""
7212   {
7213     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7214     output_asm_insn
7215       ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7216        operands);
7217     return "";
7218   }
7219   [(set_attr "length" "16")
7220    (set_attr "type" "block")
7221    (set_attr "speculation_barrier" "true")]
7222 )
7223
7224 (define_insn "despeculate_simple<ALLI:mode>"
7225   [(set (match_operand:ALLI 0 "register_operand" "=r")
7226         (unspec_volatile:ALLI
7227          [(match_operand:ALLI 1 "register_operand" "r")
7228           (use (match_operand:ALLI 2 "register_operand" ""))]
7229          UNSPECV_SPECULATION_BARRIER))]
7230   ""
7231   "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7232   [(set_attr "type" "block")
7233    (set_attr "length" "8")
7234    (set_attr "speculation_barrier" "true")]
7235 )
7236
7237 (define_insn "despeculate_simpleti"
7238   [(set (match_operand:TI 0 "register_operand" "=r")
7239         (unspec_volatile:TI
7240          [(match_operand:TI 1 "register_operand" "r")
7241           (use (match_operand:DI 2 "register_operand" ""))]
7242          UNSPECV_SPECULATION_BARRIER))]
7243   ""
7244   "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7245   [(set_attr "type" "block")
7246    (set_attr "length" "12")
7247    (set_attr "speculation_barrier" "true")]
7248 )
7249
7250 ;; Transactional Memory Extension (TME) instructions.
7251
7252 (define_insn "tstart"
7253   [(set (match_operand:DI 0 "register_operand" "=r")
7254         (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7255    (clobber (mem:BLK (scratch)))]
7256   "TARGET_TME"
7257   "tstart\\t%0"
7258   [(set_attr "type" "tme")]
7259 )
7260
7261 (define_insn "ttest"
7262   [(set (match_operand:DI 0 "register_operand" "=r")
7263         (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7264    (clobber (mem:BLK (scratch)))]
7265   "TARGET_TME"
7266   "ttest\\t%0"
7267   [(set_attr "type" "tme")]
7268 )
7269
7270 (define_insn "tcommit"
7271   [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7272    (clobber (mem:BLK (scratch)))]
7273   "TARGET_TME"
7274   "tcommit"
7275   [(set_attr "type" "tme")]
7276 )
7277
7278 (define_insn "tcancel"
7279   [(unspec_volatile:BLK
7280      [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7281    (clobber (mem:BLK (scratch)))]
7282   "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7283   "tcancel\\t#%0"
7284   [(set_attr "type" "tme")]
7285 )
7286
7287 ;; AdvSIMD Stuff
7288 (include "aarch64-simd.md")
7289
7290 ;; Atomic Operations
7291 (include "atomics.md")
7292
7293 ;; ldp/stp peephole patterns
7294 (include "aarch64-ldpstp.md")
7295
7296 ;; SVE.
7297 (include "aarch64-sve.md")
7298
7299 ;; SVE2.
7300 (include "aarch64-sve2.md")