gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2020 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (R17_REGNUM         17)
  42     (R18_REGNUM         18)
  43     (R19_REGNUM         19)
  44     (R20_REGNUM         20)
  45     (R21_REGNUM         21)
  46     (R22_REGNUM         22)
  47     (R23_REGNUM         23)
  48     (R24_REGNUM         24)
  49     (R25_REGNUM         25)
  50     (R26_REGNUM         26)
  51     (R27_REGNUM         27)
  52     (R28_REGNUM         28)
  53     (R29_REGNUM         29)
  54     (R30_REGNUM         30)
  55     (SP_REGNUM          31)
  56     (V0_REGNUM          32)
  57     (V1_REGNUM          33)
  58     (V2_REGNUM          34)
  59     (V3_REGNUM          35)
  60     (V4_REGNUM          36)
  61     (V5_REGNUM          37)
  62     (V6_REGNUM          38)
  63     (V7_REGNUM          39)
  64     (V8_REGNUM          40)
  65     (V9_REGNUM          41)
  66     (V10_REGNUM         42)
  67     (V11_REGNUM         43)
  68     (V12_REGNUM         44)
  69     (V13_REGNUM         45)
  70     (V14_REGNUM         46)
  71     (V15_REGNUM         47)
  72     (V16_REGNUM         48)
  73     (V17_REGNUM         49)
  74     (V18_REGNUM         50)
  75     (V19_REGNUM         51)
  76     (V20_REGNUM         52)
  77     (V21_REGNUM         53)
  78     (V22_REGNUM         54)
  79     (V23_REGNUM         55)
  80     (V24_REGNUM         56)
  81     (V25_REGNUM         57)
  82     (V26_REGNUM         58)
  83     (V27_REGNUM         59)
  84     (V28_REGNUM         60)
  85     (V29_REGNUM         61)
  86     (V30_REGNUM         62)
  87     (V31_REGNUM         63)
  88     (SFP_REGNUM         64)
  89     (AP_REGNUM          65)
  90     (CC_REGNUM          66)
  91     ;; Defined only to make the DWARF description simpler.
  92     (VG_REGNUM          67)
  93     (P0_REGNUM          68)
  94     (P1_REGNUM          69)
  95     (P2_REGNUM          70)
  96     (P3_REGNUM          71)
  97     (P4_REGNUM          72)
  98     (P5_REGNUM          73)
  99     (P6_REGNUM          74)
 100     (P7_REGNUM          75)
 101     (P8_REGNUM          76)
 102     (P9_REGNUM          77)
 103     (P10_REGNUM         78)
 104     (P11_REGNUM         79)
 105     (P12_REGNUM         80)
 106     (P13_REGNUM         81)
 107     (P14_REGNUM         82)
 108     (P15_REGNUM         83)
 109     (LAST_SAVED_REGNUM  83)
 110     (FFR_REGNUM         84)
 111     ;; "FFR token": a fake register used for representing the scheduling
 112     ;; restrictions on FFR-related operations.
 113     (FFRT_REGNUM        85)
 114     ;; Scratch register used by stack clash protection to calculate
 115     ;; SVE CFA offsets during probing.
 116     (STACK_CLASH_SVE_CFA_REGNUM 11)
 117     ;; Scratch registers for prologue/epilogue use.
 118     (EP0_REGNUM         12)
 119     (EP1_REGNUM         13)
 120     ;; A couple of call-clobbered registers that we need to reserve when
 121     ;; tracking speculation this is not ABI, so is subject to change.
 122     (SPECULATION_SCRATCH_REGNUM 14)
 123     (SPECULATION_TRACKER_REGNUM 15)
 124     ;; Scratch registers used in frame layout.
 125     (IP0_REGNUM         16)
 126     (IP1_REGNUM         17)
 127     (FP_REGNUM          29)
 128     (LR_REGNUM          30)
 129   ]
 130 )
 131
 132 (define_c_enum "unspec" [
 133     UNSPEC_AUTIA1716
 134     UNSPEC_AUTIB1716
 135     UNSPEC_AUTIASP
 136     UNSPEC_AUTIBSP
 137     UNSPEC_CALLEE_ABI
 138     UNSPEC_CASESI
 139     UNSPEC_CRC32B
 140     UNSPEC_CRC32CB
 141     UNSPEC_CRC32CH
 142     UNSPEC_CRC32CW
 143     UNSPEC_CRC32CX
 144     UNSPEC_CRC32H
 145     UNSPEC_CRC32W
 146     UNSPEC_CRC32X
 147     UNSPEC_FCVTZS
 148     UNSPEC_FCVTZU
 149     UNSPEC_FJCVTZS
 150     UNSPEC_FRINT32Z
 151     UNSPEC_FRINT32X
 152     UNSPEC_FRINT64Z
 153     UNSPEC_FRINT64X
 154     UNSPEC_URECPE
 155     UNSPEC_FRECPE
 156     UNSPEC_FRECPS
 157     UNSPEC_FRECPX
 158     UNSPEC_FRINTA
 159     UNSPEC_FRINTI
 160     UNSPEC_FRINTM
 161     UNSPEC_FRINTN
 162     UNSPEC_FRINTP
 163     UNSPEC_FRINTX
 164     UNSPEC_FRINTZ
 165     UNSPEC_GOTSMALLPIC
 166     UNSPEC_GOTSMALLPIC28K
 167     UNSPEC_GOTSMALLTLS
 168     UNSPEC_GOTTINYPIC
 169     UNSPEC_GOTTINYTLS
 170     UNSPEC_LD1
 171     UNSPEC_LD2
 172     UNSPEC_LD2_DREG
 173     UNSPEC_LD2_DUP
 174     UNSPEC_LD3
 175     UNSPEC_LD3_DREG
 176     UNSPEC_LD3_DUP
 177     UNSPEC_LD4
 178     UNSPEC_LD4_DREG
 179     UNSPEC_LD4_DUP
 180     UNSPEC_LD2_LANE
 181     UNSPEC_LD3_LANE
 182     UNSPEC_LD4_LANE
 183     UNSPEC_MB
 184     UNSPEC_NOP
 185     UNSPEC_PACIA1716
 186     UNSPEC_PACIB1716
 187     UNSPEC_PACIASP
 188     UNSPEC_PACIBSP
 189     UNSPEC_PRLG_STK
 190     UNSPEC_REV
 191     UNSPEC_RBIT
 192     UNSPEC_SABAL
 193     UNSPEC_SABDL2
 194     UNSPEC_SADALP
 195     UNSPEC_SCVTF
 196     UNSPEC_SISD_NEG
 197     UNSPEC_SISD_SSHL
 198     UNSPEC_SISD_USHL
 199     UNSPEC_SSHL_2S
 200     UNSPEC_ST1
 201     UNSPEC_ST2
 202     UNSPEC_ST3
 203     UNSPEC_ST4
 204     UNSPEC_ST2_LANE
 205     UNSPEC_ST3_LANE
 206     UNSPEC_ST4_LANE
 207     UNSPEC_TLS
 208     UNSPEC_TLSDESC
 209     UNSPEC_TLSLE12
 210     UNSPEC_TLSLE24
 211     UNSPEC_TLSLE32
 212     UNSPEC_TLSLE48
 213     UNSPEC_UABAL
 214     UNSPEC_UABDL2
 215     UNSPEC_UADALP
 216     UNSPEC_UCVTF
 217     UNSPEC_USHL_2S
 218     UNSPEC_VSTRUCTDUMMY
 219     UNSPEC_SSP_SYSREG
 220     UNSPEC_SP_SET
 221     UNSPEC_SP_TEST
 222     UNSPEC_RSQRT
 223     UNSPEC_RSQRTE
 224     UNSPEC_RSQRTS
 225     UNSPEC_NZCV
 226     UNSPEC_XPACLRI
 227     UNSPEC_LD1_SVE
 228     UNSPEC_ST1_SVE
 229     UNSPEC_LDNT1_SVE
 230     UNSPEC_STNT1_SVE
 231     UNSPEC_LD1RQ
 232     UNSPEC_LD1_GATHER
 233     UNSPEC_LDFF1_GATHER
 234     UNSPEC_LDNT1_GATHER
 235     UNSPEC_ST1_SCATTER
 236     UNSPEC_STNT1_SCATTER
 237     UNSPEC_PRED_X
 238     UNSPEC_PRED_Z
 239     UNSPEC_PTEST
 240     UNSPEC_PTRUE
 241     UNSPEC_UNPACKSHI
 242     UNSPEC_UNPACKUHI
 243     UNSPEC_UNPACKSLO
 244     UNSPEC_UNPACKULO
 245     UNSPEC_PACK
 246     UNSPEC_WHILEGE
 247     UNSPEC_WHILEGT
 248     UNSPEC_WHILEHI
 249     UNSPEC_WHILEHS
 250     UNSPEC_WHILELE
 251     UNSPEC_WHILELO
 252     UNSPEC_WHILELS
 253     UNSPEC_WHILELT
 254     UNSPEC_WHILERW
 255     UNSPEC_WHILEWR
 256     UNSPEC_LDN
 257     UNSPEC_STN
 258     UNSPEC_INSR
 259     UNSPEC_CLASTA
 260     UNSPEC_CLASTB
 261     UNSPEC_FADDA
 262     UNSPEC_REV_SUBREG
 263     UNSPEC_REINTERPRET
 264     UNSPEC_SPECULATION_TRACKER
 265     UNSPEC_SPECULATION_TRACKER_REV
 266     UNSPEC_COPYSIGN
 267     UNSPEC_TTEST                ; Represent transaction test.
 268     UNSPEC_UPDATE_FFR
 269     UNSPEC_UPDATE_FFRT
 270     UNSPEC_RDFFR
 271     UNSPEC_WRFFR
 272     ;; Represents an SVE-style lane index, in which the indexing applies
 273     ;; within the containing 128-bit block.
 274     UNSPEC_SVE_LANE_SELECT
 275     UNSPEC_SVE_CNT_PAT
 276     UNSPEC_SVE_PREFETCH
 277     UNSPEC_SVE_PREFETCH_GATHER
 278     UNSPEC_SVE_COMPACT
 279     UNSPEC_SVE_SPLICE
 280     UNSPEC_GEN_TAG              ; Generate a 4-bit MTE tag.
 281     UNSPEC_GEN_TAG_RND          ; Generate a random 4-bit MTE tag.
 282     UNSPEC_TAG_SPACE            ; Translate address to MTE tag address space.
 283     UNSPEC_LD1RO
 284 ])
 285
 286 (define_c_enum "unspecv" [
 287     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 288     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 289     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 290     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 291     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 292     UNSPECV_BLOCKAGE            ; Represent a blockage
 293     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 294     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 295     UNSPECV_BTI_NOARG           ; Represent BTI.
 296     UNSPECV_BTI_C               ; Represent BTI c.
 297     UNSPECV_BTI_J               ; Represent BTI j.
 298     UNSPECV_BTI_JC              ; Represent BTI jc.
 299     UNSPECV_TSTART              ; Represent transaction start.
 300     UNSPECV_TCOMMIT             ; Represent transaction commit.
 301     UNSPECV_TCANCEL             ; Represent transaction cancel.
 302     UNSPEC_RNDR                 ; Represent RNDR
 303     UNSPEC_RNDRRS               ; Represent RNDRRS
 304   ]
 305 )
 306
 307 ;; These constants are used as a const_int in various SVE unspecs
 308 ;; to indicate whether the governing predicate is known to be a PTRUE.
 309 (define_constants
 310   [; Indicates that the predicate might not be a PTRUE.
 311    (SVE_MAYBE_NOT_PTRUE 0)
 312
 313    ; Indicates that the predicate is known to be a PTRUE.
 314    (SVE_KNOWN_PTRUE 1)])
 315
 316 ;; These constants are used as a const_int in predicated SVE FP arithmetic
 317 ;; to indicate whether the operation is allowed to make additional lanes
 318 ;; active without worrying about the effect on faulting behavior.
 319 (define_constants
 320   [; Indicates either that all lanes are active or that the instruction may
 321    ; operate on inactive inputs even if doing so could induce a fault.
 322    (SVE_RELAXED_GP 0)
 323
 324    ; Indicates that some lanes might be inactive and that the instruction
 325    ; must not operate on inactive inputs if doing so could induce a fault.
 326    (SVE_STRICT_GP 1)])
 327
 328 ;; If further include files are added the defintion of MD_INCLUDES
 329 ;; must be updated.
 330
 331 (include "constraints.md")
 332 (include "predicates.md")
 333 (include "iterators.md")
 334
 335 ;; -------------------------------------------------------------------
 336 ;; Instruction types and attributes
 337 ;; -------------------------------------------------------------------
 338
 339 ; The "type" attribute is included here from AArch32 backend to be able
 340 ; to share pipeline descriptions.
 341 (include "../arm/types.md")
 342
 343 ;; It is important to set the fp or simd attributes to yes when a pattern
 344 ;; alternative uses the FP or SIMD register files, usually signified by use of
 345 ;; the 'w' constraint.  This will ensure that the alternative will be
 346 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 347 ;; architecture extensions.  If all the alternatives in a pattern use the
 348 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 349 ;; or TARGET_SIMD.
 350
 351 ;; Attributes of the architecture required to support the instruction (or
 352 ;; alternative). This attribute is used to compute attribute "enabled", use type
 353 ;; "any" to enable an alternative in all cases.
 354
 355 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
 356
 357 (define_enum_attr "arch" "arches" (const_string "any"))
 358
 359 ;; [For compatibility with Arm in pipeline models]
 360 ;; Attribute that specifies whether or not the instruction touches fp
 361 ;; registers.
 362 ;; Note that this attribute is not used anywhere in either the arm or aarch64
 363 ;; backends except in the scheduling description for xgene1.  In that
 364 ;; scheduling description this attribute is used to subclass the load_4 and
 365 ;; load_8 types.
 366 (define_attr "fp" "no,yes"
 367   (if_then_else
 368     (eq_attr "arch" "fp")
 369     (const_string "yes")
 370     (const_string "no")))
 371
 372 (define_attr "arch_enabled" "no,yes"
 373   (if_then_else
 374     (ior
 375         (eq_attr "arch" "any")
 376
 377         (and (eq_attr "arch" "rcpc8_4")
 378              (match_test "AARCH64_ISA_RCPC8_4"))
 379
 380         (and (eq_attr "arch" "fp")
 381              (match_test "TARGET_FLOAT"))
 382
 383         (and (eq_attr "arch" "simd")
 384              (match_test "TARGET_SIMD"))
 385
 386         (and (eq_attr "arch" "fp16")
 387              (match_test "TARGET_FP_F16INST"))
 388
 389         (and (eq_attr "arch" "sve")
 390              (match_test "TARGET_SVE")))
 391     (const_string "yes")
 392     (const_string "no")))
 393
 394 ;; Attribute that controls whether an alternative is enabled or not.
 395 ;; Currently it is only used to disable alternatives which touch fp or simd
 396 ;; registers when -mgeneral-regs-only is specified or to require a special
 397 ;; architecture support.
 398 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
 399
 400 ;; Attribute that specifies whether we are dealing with a branch to a
 401 ;; label that is far away, i.e. further away than the maximum/minimum
 402 ;; representable in a signed 21-bits number.
 403 ;; 0 :=: no
 404 ;; 1 :=: yes
 405 (define_attr "far_branch" "" (const_int 0))
 406
 407 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 408 (define_attr "movprfx" "no,yes" (const_string "no"))
 409
 410 ;; Attribute to specify that an alternative has the length of a single
 411 ;; instruction plus a speculation barrier.
 412 (define_attr "sls_length" "none,retbr,casesi" (const_string "none"))
 413
 414 (define_attr "length" ""
 415   (cond [(eq_attr "movprfx" "yes")
 416            (const_int 8)
 417
 418          (eq_attr "sls_length" "retbr")
 419            (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 4)
 420                   (match_test "TARGET_SB") (const_int 8)]
 421                  (const_int 12))
 422
 423          (eq_attr "sls_length" "casesi")
 424            (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 16)
 425                   (match_test "TARGET_SB") (const_int 20)]
 426                  (const_int 24))
 427         ]
 428           (const_int 4)))
 429
 430 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 431 ;; no predicated insns.
 432 (define_attr "predicated" "yes,no" (const_string "no"))
 433
 434 ;; Set to true on an insn that requires the speculation tracking state to be
 435 ;; in the tracking register before the insn issues.  Otherwise the compiler
 436 ;; may chose to hold the tracking state encoded in SP.
 437 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 438
 439 ;; -------------------------------------------------------------------
 440 ;; Pipeline descriptions and scheduling
 441 ;; -------------------------------------------------------------------
 442
 443 ;; Processor types.
 444 (include "aarch64-tune.md")
 445
 446 ;; Scheduling
 447 (include "../arm/cortex-a53.md")
 448 (include "../arm/cortex-a57.md")
 449 (include "../arm/exynos-m1.md")
 450 (include "falkor.md")
 451 (include "saphira.md")
 452 (include "thunderx.md")
 453 (include "../arm/xgene1.md")
 454 (include "thunderx2t99.md")
 455 (include "tsv110.md")
 456 (include "thunderx3t110.md")
 457
 458 ;; -------------------------------------------------------------------
 459 ;; Jumps and other miscellaneous insns
 460 ;; -------------------------------------------------------------------
 461
 462 (define_insn "indirect_jump"
 463   [(set (pc) (match_operand 0 "pmode_register_operand" "r"))]
 464   ""
 465   {
 466     output_asm_insn ("br\\t%0", operands);
 467     return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
 468   }
 469   [(set_attr "type" "branch")
 470    (set_attr "sls_length" "retbr")]
 471 )
 472
 473 (define_insn "jump"
 474   [(set (pc) (label_ref (match_operand 0 "" "")))]
 475   ""
 476   "b\\t%l0"
 477   [(set_attr "type" "branch")]
 478 )
 479
 480 (define_expand "cbranch<mode>4"
 481   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 482                             [(match_operand:GPI 1 "register_operand")
 483                              (match_operand:GPI 2 "aarch64_plus_operand")])
 484                            (label_ref (match_operand 3 "" ""))
 485                            (pc)))]
 486   ""
 487   "
 488   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 489                                          operands[2]);
 490   operands[2] = const0_rtx;
 491   "
 492 )
 493
 494 (define_expand "cbranch<mode>4"
 495   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 496                             [(match_operand:GPF 1 "register_operand")
 497                              (match_operand:GPF 2 "aarch64_fp_compare_operand")])
 498                            (label_ref (match_operand 3 "" ""))
 499                            (pc)))]
 500   ""
 501   "
 502   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 503                                          operands[2]);
 504   operands[2] = const0_rtx;
 505   "
 506 )
 507
 508 (define_expand "cbranchcc4"
 509   [(set (pc) (if_then_else
 510               (match_operator 0 "aarch64_comparison_operator"
 511                [(match_operand 1 "cc_register")
 512                 (match_operand 2 "const0_operand")])
 513               (label_ref (match_operand 3 "" ""))
 514               (pc)))]
 515   ""
 516   "")
 517
 518 (define_insn "@ccmp<CC_ONLY:mode><GPI:mode>"
 519   [(set (match_operand:CC_ONLY 1 "cc_register" "")
 520         (if_then_else:CC_ONLY
 521           (match_operator 4 "aarch64_comparison_operator"
 522            [(match_operand 0 "cc_register" "")
 523             (const_int 0)])
 524           (compare:CC_ONLY
 525             (match_operand:GPI 2 "register_operand" "r,r,r")
 526             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 527           (unspec:CC_ONLY
 528             [(match_operand 5 "immediate_operand")]
 529             UNSPEC_NZCV)))]
 530   ""
 531   "@
 532    ccmp\\t%<w>2, %<w>3, %k5, %m4
 533    ccmp\\t%<w>2, %3, %k5, %m4
 534    ccmn\\t%<w>2, #%n3, %k5, %m4"
 535   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 536 )
 537
 538 (define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>"
 539   [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
 540         (if_then_else:CCFP_CCFPE
 541           (match_operator 4 "aarch64_comparison_operator"
 542            [(match_operand 0 "cc_register" "")
 543             (const_int 0)])
 544           (compare:CCFP_CCFPE
 545             (match_operand:GPF 2 "register_operand" "w")
 546             (match_operand:GPF 3 "register_operand" "w"))
 547           (unspec:CCFP_CCFPE
 548             [(match_operand 5 "immediate_operand")]
 549             UNSPEC_NZCV)))]
 550   "TARGET_FLOAT"
 551   "fccmp<e>\\t%<s>2, %<s>3, %k5, %m4"
 552   [(set_attr "type" "fccmp<s>")]
 553 )
 554
 555 (define_insn "@ccmp<CC_ONLY:mode><GPI:mode>_rev"
 556   [(set (match_operand:CC_ONLY 1 "cc_register" "")
 557         (if_then_else:CC_ONLY
 558           (match_operator 4 "aarch64_comparison_operator"
 559            [(match_operand 0 "cc_register" "")
 560             (const_int 0)])
 561           (unspec:CC_ONLY
 562             [(match_operand 5 "immediate_operand")]
 563             UNSPEC_NZCV)
 564           (compare:CC_ONLY
 565             (match_operand:GPI 2 "register_operand" "r,r,r")
 566             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))))]
 567   ""
 568   "@
 569    ccmp\\t%<w>2, %<w>3, %k5, %M4
 570    ccmp\\t%<w>2, %3, %k5, %M4
 571    ccmn\\t%<w>2, #%n3, %k5, %M4"
 572   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 573 )
 574
 575 (define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>_rev"
 576   [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
 577         (if_then_else:CCFP_CCFPE
 578           (match_operator 4 "aarch64_comparison_operator"
 579            [(match_operand 0 "cc_register" "")
 580             (const_int 0)])
 581           (unspec:CCFP_CCFPE
 582             [(match_operand 5 "immediate_operand")]
 583             UNSPEC_NZCV)
 584           (compare:CCFP_CCFPE
 585             (match_operand:GPF 2 "register_operand" "w")
 586             (match_operand:GPF 3 "register_operand" "w"))))]
 587   "TARGET_FLOAT"
 588   "fccmp<e>\\t%<s>2, %<s>3, %k5, %M4"
 589   [(set_attr "type" "fccmp<s>")]
 590 )
 591
 592 ;; Expansion of signed mod by a power of 2 using CSNEG.
 593 ;; For x0 % n where n is a power of 2 produce:
 594 ;; negs   x1, x0
 595 ;; and    x0, x0, #(n - 1)
 596 ;; and    x1, x1, #(n - 1)
 597 ;; csneg  x0, x0, x1, mi
 598
 599 (define_expand "mod<mode>3"
 600   [(match_operand:GPI 0 "register_operand")
 601    (match_operand:GPI 1 "register_operand")
 602    (match_operand:GPI 2 "const_int_operand")]
 603   ""
 604   {
 605     HOST_WIDE_INT val = INTVAL (operands[2]);
 606
 607     if (val <= 0
 608        || exact_log2 (val) <= 0
 609        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 610       FAIL;
 611
 612     rtx mask = GEN_INT (val - 1);
 613
 614     /* In the special case of x0 % 2 we can do the even shorter:
 615         cmp    x0, xzr
 616         and    x0, x0, 1
 617         cneg   x0, x0, lt.  */
 618     if (val == 2)
 619       {
 620         rtx masked = gen_reg_rtx (<MODE>mode);
 621         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 622         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 623         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 624         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 625         DONE;
 626       }
 627
 628     rtx neg_op = gen_reg_rtx (<MODE>mode);
 629     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 630
 631     /* Extract the condition register and mode.  */
 632     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 633     rtx cc_reg = SET_DEST (cmp);
 634     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 635
 636     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 637     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 638
 639     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 640     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 641
 642     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 643                                        masked_neg, masked_pos));
 644     DONE;
 645   }
 646 )
 647
 648 (define_insn "condjump"
 649   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 650                             [(match_operand 1 "cc_register" "") (const_int 0)])
 651                            (label_ref (match_operand 2 "" ""))
 652                            (pc)))]
 653   ""
 654   {
 655     /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
 656        but the "." is required for SVE conditions.  */
 657     bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
 658     if (get_attr_length (insn) == 8)
 659       return aarch64_gen_far_branch (operands, 2, "Lbcond",
 660                                      use_dot_p ? "b.%M0\\t" : "b%M0\\t");
 661     else
 662       return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
 663   }
 664   [(set_attr "type" "branch")
 665    (set (attr "length")
 666         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 667                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 668                       (const_int 4)
 669                       (const_int 8)))
 670    (set (attr "far_branch")
 671         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 672                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 673                       (const_int 0)
 674                       (const_int 1)))]
 675 )
 676
 677 ;; For a 24-bit immediate CST we can optimize the compare for equality
 678 ;; and branch sequence from:
 679 ;;      mov     x0, #imm1
 680 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 681 ;;      cmp     x1, x0
 682 ;;      b<ne,eq> .Label
 683 ;; into the shorter:
 684 ;;      sub     x0, x1, #(CST & 0xfff000)
 685 ;;      subs    x0, x0, #(CST & 0x000fff)
 686 ;;      b<ne,eq> .Label
 687 (define_insn_and_split "*compare_condjump<GPI:mode>"
 688   [(set (pc) (if_then_else (EQL
 689                               (match_operand:GPI 0 "register_operand" "r")
 690                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 691                            (label_ref:P (match_operand 2 "" ""))
 692                            (pc)))]
 693   "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
 694    && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
 695    && !reload_completed"
 696   "#"
 697   "&& true"
 698   [(const_int 0)]
 699   {
 700     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 701     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 702     rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
 703     emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 704     emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 705     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 706     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
 707                                   cc_reg, const0_rtx);
 708     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 709     DONE;
 710   }
 711 )
 712
 713 (define_expand "casesi"
 714   [(match_operand:SI 0 "register_operand")      ; Index
 715    (match_operand:SI 1 "const_int_operand")     ; Lower bound
 716    (match_operand:SI 2 "const_int_operand")     ; Total range
 717    (match_operand:DI 3 "" "")                   ; Table label
 718    (match_operand:DI 4 "" "")]                  ; Out of range label
 719   ""
 720   {
 721     if (operands[1] != const0_rtx)
 722       {
 723         rtx reg = gen_reg_rtx (SImode);
 724
 725         /* Canonical RTL says that if you have:
 726
 727            (minus (X) (CONST))
 728
 729            then this should be emitted as:
 730
 731            (plus (X) (-CONST))
 732
 733            The use of trunc_int_for_mode ensures that the resulting
 734            constant can be represented in SImode, this is important
 735            for the corner case where operand[1] is INT_MIN.  */
 736
 737         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 738
 739         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 740               (operands[1], SImode))
 741           operands[1] = force_reg (SImode, operands[1]);
 742         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 743         operands[0] = reg;
 744       }
 745
 746     if (!aarch64_plus_operand (operands[2], SImode))
 747       operands[2] = force_reg (SImode, operands[2]);
 748     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 749                                                  const0_rtx),
 750                                     operands[0], operands[2], operands[4]));
 751
 752     operands[2] = force_reg (Pmode, gen_rtx_LABEL_REF (Pmode, operands[3]));
 753     operands[2]
 754       = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
 755                         UNSPEC_CASESI);
 756     operands[2] = gen_rtx_MEM (Pmode, operands[2]);
 757     MEM_READONLY_P (operands[2]) = 1;
 758     MEM_NOTRAP_P (operands[2]) = 1;
 759
 760     emit_jump_insn (gen_casesi_dispatch (Pmode, operands[2], operands[3]));
 761     DONE;
 762   }
 763 )
 764
 765 (define_expand "@casesi_dispatch<mode>"
 766   [(parallel
 767     [(set (pc) (match_operand:ADDR 0 ""))
 768      (clobber (reg:CC CC_REGNUM))
 769      (clobber (match_scratch:ADDR 2))
 770      (clobber (match_scratch:ADDR 3))
 771      (use (label_ref:ADDR (match_operand 1 "")))])]
 772   "")
 773
 774 (define_insn "*casesi_dispatch"
 775   [(parallel
 776     [(set (pc)
 777           (mem:ADDR (unspec [(match_operand:ADDR 0 "register_operand" "r")
 778                              (match_operand:SI 1 "register_operand" "r")]
 779                         UNSPEC_CASESI)))
 780      (clobber (reg:CC CC_REGNUM))
 781      (clobber (match_scratch:ADDR 3 "=r"))
 782      (clobber (match_scratch:ADDR 4 "=r"))
 783      (use (label_ref:ADDR (match_operand 2 "" "")))])]
 784   ""
 785   "*
 786   return aarch64_output_casesi (operands);
 787   "
 788   [(set_attr "sls_length" "casesi")
 789    (set_attr "type" "branch")]
 790 )
 791
 792 (define_insn "nop"
 793   [(unspec[(const_int 0)] UNSPEC_NOP)]
 794   ""
 795   "nop"
 796   [(set_attr "type" "no_insn")]
 797 )
 798
 799 (define_insn "prefetch"
 800   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 801             (match_operand:QI 1 "const_int_operand" "")
 802             (match_operand:QI 2 "const_int_operand" ""))]
 803   ""
 804   {
 805     const char * pftype[2][4] =
 806     {
 807       {"prfm\\tPLDL1STRM, %0",
 808        "prfm\\tPLDL3KEEP, %0",
 809        "prfm\\tPLDL2KEEP, %0",
 810        "prfm\\tPLDL1KEEP, %0"},
 811       {"prfm\\tPSTL1STRM, %0",
 812        "prfm\\tPSTL3KEEP, %0",
 813        "prfm\\tPSTL2KEEP, %0",
 814        "prfm\\tPSTL1KEEP, %0"},
 815     };
 816
 817     int locality = INTVAL (operands[2]);
 818
 819     gcc_assert (IN_RANGE (locality, 0, 3));
 820
 821     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 822        the address into a DImode MEM so that aarch64_print_operand knows
 823        how to print it.  */
 824     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 825     return pftype[INTVAL(operands[1])][locality];
 826   }
 827   [(set_attr "type" "load_4")]
 828 )
 829
 830 (define_insn "trap"
 831   [(trap_if (const_int 1) (const_int 8))]
 832   ""
 833   "brk #1000"
 834   [(set_attr "type" "trap")])
 835
 836 (define_expand "prologue"
 837   [(clobber (const_int 0))]
 838   ""
 839   "
 840   aarch64_expand_prologue ();
 841   DONE;
 842   "
 843 )
 844
 845 (define_expand "epilogue"
 846   [(clobber (const_int 0))]
 847   ""
 848   "
 849   aarch64_expand_epilogue (false);
 850   DONE;
 851   "
 852 )
 853
 854 (define_expand "sibcall_epilogue"
 855   [(clobber (const_int 0))]
 856   ""
 857   "
 858   aarch64_expand_epilogue (true);
 859   DONE;
 860   "
 861 )
 862
 863 (define_insn "*do_return"
 864   [(return)]
 865   ""
 866   {
 867     const char *ret = NULL;
 868     if (aarch64_return_address_signing_enabled ()
 869         && TARGET_ARMV8_3
 870         && !crtl->calls_eh_return)
 871       {
 872         if (aarch64_ra_sign_key == AARCH64_KEY_B)
 873           ret = "retab";
 874         else
 875           ret = "retaa";
 876       }
 877     else
 878       ret = "ret";
 879     output_asm_insn (ret, operands);
 880     return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
 881   }
 882   [(set_attr "type" "branch")
 883    (set_attr "sls_length" "retbr")]
 884 )
 885
 886 (define_expand "return"
 887   [(simple_return)]
 888   "aarch64_use_return_insn_p ()"
 889   ""
 890 )
 891
 892 (define_insn "simple_return"
 893   [(simple_return)]
 894   ""
 895   {
 896     output_asm_insn ("ret", operands);
 897     return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
 898   }
 899   [(set_attr "type" "branch")
 900    (set_attr "sls_length" "retbr")]
 901 )
 902
 903 (define_insn "*cb<optab><mode>1"
 904   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 905                                 (const_int 0))
 906                            (label_ref (match_operand 1 "" ""))
 907                            (pc)))]
 908   "!aarch64_track_speculation"
 909   {
 910     if (get_attr_length (insn) == 8)
 911       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 912     else
 913       return "<cbz>\\t%<w>0, %l1";
 914   }
 915   [(set_attr "type" "branch")
 916    (set (attr "length")
 917         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 918                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 919                       (const_int 4)
 920                       (const_int 8)))
 921    (set (attr "far_branch")
 922         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 923                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 924                       (const_int 0)
 925                       (const_int 1)))]
 926 )
 927
 928 (define_insn "*tb<optab><mode>1"
 929   [(set (pc) (if_then_else
 930               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 931                                     (const_int 1)
 932                                     (match_operand 1
 933                                       "aarch64_simd_shift_imm_<mode>" "n"))
 934                    (const_int 0))
 935              (label_ref (match_operand 2 "" ""))
 936              (pc)))
 937    (clobber (reg:CC CC_REGNUM))]
 938   "!aarch64_track_speculation"
 939   {
 940     if (get_attr_length (insn) == 8)
 941       {
 942         if (get_attr_far_branch (insn) == 1)
 943           return aarch64_gen_far_branch (operands, 2, "Ltb",
 944                                          "<inv_tb>\\t%<w>0, %1, ");
 945         else
 946           {
 947             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 948             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 949           }
 950       }
 951     else
 952       return "<tbz>\t%<w>0, %1, %l2";
 953   }
 954   [(set_attr "type" "branch")
 955    (set (attr "length")
 956         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 957                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 958                       (const_int 4)
 959                       (const_int 8)))
 960    (set (attr "far_branch")
 961         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 962                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 963                       (const_int 0)
 964                       (const_int 1)))]
 965
 966 )
 967
 968 (define_insn "*cb<optab><mode>1"
 969   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 970                                  (const_int 0))
 971                            (label_ref (match_operand 1 "" ""))
 972                            (pc)))
 973    (clobber (reg:CC CC_REGNUM))]
 974   "!aarch64_track_speculation"
 975   {
 976     if (get_attr_length (insn) == 8)
 977       {
 978         if (get_attr_far_branch (insn) == 1)
 979           return aarch64_gen_far_branch (operands, 1, "Ltb",
 980                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 981         else
 982           {
 983             char buf[64];
 984             uint64_t val = ((uint64_t) 1)
 985                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 986             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 987             output_asm_insn (buf, operands);
 988             return "<bcond>\t%l1";
 989           }
 990       }
 991     else
 992       return "<tbz>\t%<w>0, <sizem1>, %l1";
 993   }
 994   [(set_attr "type" "branch")
 995    (set (attr "length")
 996         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 997                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 998                       (const_int 4)
 999                       (const_int 8)))
1000    (set (attr "far_branch")
1001         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
1002                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
1003                       (const_int 0)
1004                       (const_int 1)))]
1005 )
1006
1007 ;; -------------------------------------------------------------------
1008 ;; Subroutine calls and sibcalls
1009 ;; -------------------------------------------------------------------
1010
1011 (define_expand "call"
1012   [(parallel
1013      [(call (match_operand 0 "memory_operand")
1014             (match_operand 1 "general_operand"))
1015       (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1016       (clobber (reg:DI LR_REGNUM))])]
1017   ""
1018   "
1019   {
1020     aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
1021     DONE;
1022   }"
1023 )
1024
1025 (define_insn "*call_insn"
1026   [(call (mem:DI (match_operand:ADDR 0 "aarch64_call_insn_operand" "Ucr, Usf"))
1027          (match_operand 1 "" ""))
1028    (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1029    (clobber (reg:ADDR LR_REGNUM))]
1030   ""
1031   "@
1032   * return aarch64_indirect_call_asm (operands[0]);
1033   bl\\t%c0"
1034   [(set_attr "type" "call, call")])
1035
1036 (define_expand "call_value"
1037   [(parallel
1038      [(set (match_operand 0 "")
1039            (call (match_operand 1 "memory_operand")
1040                  (match_operand 2 "general_operand")))
1041      (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1042      (clobber (reg:DI LR_REGNUM))])]
1043   ""
1044   "
1045   {
1046     aarch64_expand_call (operands[0], operands[1], operands[3], false);
1047     DONE;
1048   }"
1049 )
1050
1051 (define_insn "*call_value_insn"
1052   [(set (match_operand 0 "" "")
1053         (call (mem:DI (match_operand:ADDR 1 "aarch64_call_insn_operand" "Ucr, Usf"))
1054                       (match_operand 2 "" "")))
1055    (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1056    (clobber (reg:ADDR LR_REGNUM))]
1057   ""
1058   "@
1059   * return aarch64_indirect_call_asm (operands[1]);
1060   bl\\t%c1"
1061   [(set_attr "type" "call, call")]
1062 )
1063
1064 (define_expand "sibcall"
1065   [(parallel
1066      [(call (match_operand 0 "memory_operand")
1067             (match_operand 1 "general_operand"))
1068       (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1069       (return)])]
1070   ""
1071   {
1072     aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
1073     DONE;
1074   }
1075 )
1076
1077 (define_expand "sibcall_value"
1078   [(parallel
1079      [(set (match_operand 0 "")
1080            (call (match_operand 1 "memory_operand")
1081                  (match_operand 2 "general_operand")))
1082       (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1083       (return)])]
1084   ""
1085   {
1086     aarch64_expand_call (operands[0], operands[1], operands[3], true);
1087     DONE;
1088   }
1089 )
1090
1091 (define_insn "*sibcall_insn"
1092   [(call (mem:DI (match_operand:ADDR 0 "aarch64_call_insn_operand" "Ucs, Usf"))
1093          (match_operand 1 ""))
1094    (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1095    (return)]
1096   "SIBLING_CALL_P (insn)"
1097   {
1098     if (which_alternative == 0)
1099       {
1100         output_asm_insn ("br\\t%0", operands);
1101         return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1102       }
1103     return "b\\t%c0";
1104   }
1105   [(set_attr "type" "branch, branch")
1106    (set_attr "sls_length" "retbr,none")]
1107 )
1108
1109 (define_insn "*sibcall_value_insn"
1110   [(set (match_operand 0 "")
1111         (call (mem:DI
1112                 (match_operand:ADDR 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1113               (match_operand 2 "")))
1114    (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1115    (return)]
1116   "SIBLING_CALL_P (insn)"
1117   {
1118     if (which_alternative == 0)
1119       {
1120         output_asm_insn ("br\\t%1", operands);
1121         return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1122       }
1123     return "b\\t%c1";
1124   }
1125   [(set_attr "type" "branch, branch")
1126    (set_attr "sls_length" "retbr,none")]
1127 )
1128
1129 ;; Call subroutine returning any type.
1130
1131 (define_expand "untyped_call"
1132   [(parallel [(call (match_operand 0 "")
1133                     (const_int 0))
1134               (match_operand 1 "")
1135               (match_operand 2 "")])]
1136   ""
1137 {
1138   int i;
1139
1140   /* Untyped calls always use the default ABI.  It's only possible to use
1141      ABI variants if we know the type of the target function.  */
1142   emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));
1143
1144   for (i = 0; i < XVECLEN (operands[2], 0); i++)
1145     {
1146       rtx set = XVECEXP (operands[2], 0, i);
1147       emit_move_insn (SET_DEST (set), SET_SRC (set));
1148     }
1149
1150   /* The optimizer does not know that the call sets the function value
1151      registers we stored in the result block.  We avoid problems by
1152      claiming that all hard registers are used and clobbered at this
1153      point.  */
1154   emit_insn (gen_blockage ());
1155   DONE;
1156 })
1157
1158 ;; -------------------------------------------------------------------
1159 ;; Moves
1160 ;; -------------------------------------------------------------------
1161
1162 (define_expand "mov<mode>"
1163   [(set (match_operand:SHORT 0 "nonimmediate_operand")
1164         (match_operand:SHORT 1 "general_operand"))]
1165   ""
1166   "
1167     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1168       operands[1] = force_reg (<MODE>mode, operands[1]);
1169
1170     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1171       {
1172         aarch64_expand_mov_immediate (operands[0], operands[1]);
1173         DONE;
1174       }
1175   "
1176 )
1177
1178 (define_insn "*mov<mode>_aarch64"
1179   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
1180         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1181   "(register_operand (operands[0], <MODE>mode)
1182     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1183 {
1184    switch (which_alternative)
1185      {
1186      case 0:
1187        return "mov\t%w0, %w1";
1188      case 1:
1189        return "mov\t%w0, %1";
1190      case 2:
1191        return aarch64_output_scalar_simd_mov_immediate (operands[1],
1192                                                         <MODE>mode);
1193      case 3:
1194        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1195      case 4:
1196        return "ldr<size>\t%w0, %1";
1197      case 5:
1198        return "ldr\t%<size>0, %1";
1199      case 6:
1200        return "str<size>\t%w1, %0";
1201      case 7:
1202        return "str\t%<size>1, %0";
1203      case 8:
1204        return "umov\t%w0, %1.<v>[0]";
1205      case 9:
1206        return "dup\t%0.<Vallxd>, %w1";
1207      case 10:
1208        return "dup\t%<Vetype>0, %1.<v>[0]";
1209      default:
1210        gcc_unreachable ();
1211      }
1212 }
1213   ;; The "mov_imm" type for CNT is just a placeholder.
1214   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1215                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1216    (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1217 )
1218
1219 (define_expand "mov<mode>"
1220   [(set (match_operand:GPIC 0 "nonimmediate_operand")
1221         (match_operand:GPIC 1 "general_operand"))]
1222   ""
1223   "
1224     if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1225         && CONST_INT_P (operands[1]) && <MODE>mode == DImode
1226         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1227       DONE;
1228
1229     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1230       operands[1] = force_reg (<MODE>mode, operands[1]);
1231
1232     /* FIXME: RR we still need to fix up what we are doing with
1233        symbol_refs and other types of constants.  */
1234     if (CONSTANT_P (operands[1])
1235         && !CONST_INT_P (operands[1]))
1236      {
1237        aarch64_expand_mov_immediate (operands[0], operands[1]);
1238        DONE;
1239      }
1240   "
1241 )
1242
1243 (define_insn_and_split "*movsi_aarch64"
1244   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1245         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1246   "(register_operand (operands[0], SImode)
1247     || aarch64_reg_or_zero (operands[1], SImode))"
1248   "@
1249    mov\\t%w0, %w1
1250    mov\\t%w0, %w1
1251    mov\\t%w0, %w1
1252    mov\\t%w0, %1
1253    #
1254    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1255    ldr\\t%w0, %1
1256    ldr\\t%s0, %1
1257    str\\t%w1, %0
1258    str\\t%s1, %0
1259    adr\\t%x0, %c1
1260    adrp\\t%x0, %A1
1261    fmov\\t%s0, %w1
1262    fmov\\t%w0, %s1
1263    fmov\\t%s0, %s1
1264    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1265   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1266     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1267    [(const_int 0)]
1268    "{
1269        aarch64_expand_mov_immediate (operands[0], operands[1]);
1270        DONE;
1271     }"
1272   ;; The "mov_imm" type for CNT is just a placeholder.
1273   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1274                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1275    (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1276 )
1277
1278 (define_insn_and_split "*movdi_aarch64"
1279   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1280         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1281   "(register_operand (operands[0], DImode)
1282     || aarch64_reg_or_zero (operands[1], DImode))"
1283   "@
1284    mov\\t%x0, %x1
1285    mov\\t%0, %x1
1286    mov\\t%x0, %1
1287    mov\\t%x0, %1
1288    mov\\t%w0, %1
1289    #
1290    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1291    ldr\\t%x0, %1
1292    ldr\\t%d0, %1
1293    str\\t%x1, %0
1294    str\\t%d1, %0
1295    adr\\t%x0, %c1
1296    adrp\\t%x0, %A1
1297    fmov\\t%d0, %x1
1298    fmov\\t%x0, %d1
1299    fmov\\t%d0, %d1
1300    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1301    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1302     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1303    [(const_int 0)]
1304    "{
1305        aarch64_expand_mov_immediate (operands[0], operands[1]);
1306        DONE;
1307     }"
1308   ;; The "mov_imm" type for CNTD is just a placeholder.
1309   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1310                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1311                      neon_move")
1312    (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1313 )
1314
1315 (define_insn "insv_imm<mode>"
1316   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1317                           (const_int 16)
1318                           (match_operand:GPI 1 "const_int_operand" "n"))
1319         (match_operand:GPI 2 "const_int_operand" "n"))]
1320   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1321    && UINTVAL (operands[1]) % 16 == 0"
1322   "movk\\t%<w>0, %X2, lsl %1"
1323   [(set_attr "type" "mov_imm")]
1324 )
1325
1326 ;; Match MOVK as a normal AND and IOR operation.
1327 (define_insn "aarch64_movk<mode>"
1328   [(set (match_operand:GPI 0 "register_operand" "=r")
1329         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
1330                           (match_operand:GPI 2 "const_int_operand"))
1331                  (match_operand:GPI 3 "const_int_operand")))]
1332   "aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1333                        rtx_mode_t (operands[3], <MODE>mode)) >= 0"
1334   {
1335     int shift = aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1336                                     rtx_mode_t (operands[3], <MODE>mode));
1337     operands[2] = gen_int_mode (UINTVAL (operands[3]) >> shift, SImode);
1338     operands[3] = gen_int_mode (shift, SImode);
1339     return "movk\\t%<w>0, #%X2, lsl %3";
1340   }
1341   [(set_attr "type" "mov_imm")]
1342 )
1343
1344 (define_expand "movti"
1345   [(set (match_operand:TI 0 "nonimmediate_operand")
1346         (match_operand:TI 1 "general_operand"))]
1347   ""
1348   "
1349     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1350       operands[1] = force_reg (TImode, operands[1]);
1351
1352     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1353       {
1354         emit_move_insn (gen_lowpart (DImode, operands[0]),
1355                         gen_lowpart (DImode, operands[1]));
1356         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1357         DONE;
1358       }
1359   "
1360 )
1361
1362 (define_insn "*movti_aarch64"
1363   [(set (match_operand:TI 0
1364          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1365         (match_operand:TI 1
1366          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1367   "(register_operand (operands[0], TImode)
1368     || aarch64_reg_or_zero (operands[1], TImode))"
1369   "@
1370    #
1371    #
1372    #
1373    mov\\t%0.16b, %1.16b
1374    ldp\\t%0, %H0, %1
1375    stp\\t%1, %H1, %0
1376    stp\\txzr, xzr, %0
1377    ldr\\t%q0, %1
1378    str\\t%q1, %0"
1379   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1380                              load_16,store_16,store_16,\
1381                              load_16,store_16")
1382    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1383    (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1384 )
1385
1386 ;; Split a TImode register-register or register-immediate move into
1387 ;; its component DImode pieces, taking care to handle overlapping
1388 ;; source and dest registers.
1389 (define_split
1390    [(set (match_operand:TI 0 "register_operand" "")
1391          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1392   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1393   [(const_int 0)]
1394 {
1395   aarch64_split_128bit_move (operands[0], operands[1]);
1396   DONE;
1397 })
1398
1399 (define_expand "mov<mode>"
1400   [(set (match_operand:GPF_TF_F16_MOV 0 "nonimmediate_operand")
1401         (match_operand:GPF_TF_F16_MOV 1 "general_operand"))]
1402   ""
1403   {
1404     if (!TARGET_FLOAT)
1405       {
1406         aarch64_err_no_fpadvsimd (<MODE>mode);
1407         machine_mode intmode
1408           = int_mode_for_size (GET_MODE_BITSIZE (<MODE>mode), 0).require ();
1409         emit_move_insn (gen_lowpart (intmode, operands[0]),
1410                         gen_lowpart (intmode, operands[1]));
1411         DONE;
1412       }
1413
1414     if (GET_CODE (operands[0]) == MEM
1415         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1416               && aarch64_float_const_zero_rtx_p (operands[1])))
1417       operands[1] = force_reg (<MODE>mode, operands[1]);
1418   }
1419 )
1420
1421 (define_insn "*mov<mode>_aarch64"
1422   [(set (match_operand:HFBF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1423         (match_operand:HFBF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1424   "TARGET_FLOAT && (register_operand (operands[0], <MODE>mode)
1425     || aarch64_reg_or_fp_zero (operands[1], <MODE>mode))"
1426   "@
1427    movi\\t%0.4h, #0
1428    fmov\\t%h0, %w1
1429    dup\\t%w0.4h, %w1
1430    umov\\t%w0, %1.h[0]
1431    mov\\t%0.h[0], %1.h[0]
1432    fmov\\t%h0, %1
1433    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1434    ldr\\t%h0, %1
1435    str\\t%h1, %0
1436    ldrh\\t%w0, %1
1437    strh\\t%w1, %0
1438    mov\\t%w0, %w1"
1439   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1440                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1441    (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1442 )
1443
1444 (define_insn "*movsf_aarch64"
1445   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1446         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1447   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1448     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1449   "@
1450    movi\\t%0.2s, #0
1451    fmov\\t%s0, %w1
1452    fmov\\t%w0, %s1
1453    fmov\\t%s0, %s1
1454    fmov\\t%s0, %1
1455    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1456    ldr\\t%s0, %1
1457    str\\t%s1, %0
1458    ldr\\t%w0, %1
1459    str\\t%w1, %0
1460    mov\\t%w0, %w1
1461    mov\\t%w0, %1"
1462   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1463                      f_loads,f_stores,load_4,store_4,mov_reg,\
1464                      fconsts")
1465    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1466 )
1467
1468 (define_insn "*movdf_aarch64"
1469   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1470         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1471   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1472     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1473   "@
1474    movi\\t%d0, #0
1475    fmov\\t%d0, %x1
1476    fmov\\t%x0, %d1
1477    fmov\\t%d0, %d1
1478    fmov\\t%d0, %1
1479    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1480    ldr\\t%d0, %1
1481    str\\t%d1, %0
1482    ldr\\t%x0, %1
1483    str\\t%x1, %0
1484    mov\\t%x0, %x1
1485    mov\\t%x0, %1"
1486   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1487                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1488                      fconstd")
1489    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1490 )
1491
1492 (define_split
1493   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1494         (match_operand:GPF_HF 1 "general_operand"))]
1495   "can_create_pseudo_p ()
1496    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1497    && !aarch64_float_const_representable_p (operands[1])
1498    &&  aarch64_float_const_rtx_p (operands[1])"
1499   [(const_int 0)]
1500   {
1501     unsigned HOST_WIDE_INT ival;
1502     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1503       FAIL;
1504
1505     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1506     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1507     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1508     DONE;
1509   }
1510 )
1511
1512 (define_insn "*movtf_aarch64"
1513   [(set (match_operand:TF 0
1514          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1515         (match_operand:TF 1
1516          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1517   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1518     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1519   "@
1520    mov\\t%0.16b, %1.16b
1521    #
1522    #
1523    #
1524    movi\\t%0.2d, #0
1525    fmov\\t%s0, wzr
1526    ldr\\t%q0, %1
1527    str\\t%q1, %0
1528    ldp\\t%0, %H0, %1
1529    stp\\t%1, %H1, %0
1530    stp\\txzr, xzr, %0"
1531   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1532                      f_loadd,f_stored,load_16,store_16,store_16")
1533    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1534    (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1535 )
1536
1537 (define_split
1538    [(set (match_operand:TF 0 "register_operand" "")
1539          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1540   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1541   [(const_int 0)]
1542   {
1543     aarch64_split_128bit_move (operands[0], operands[1]);
1544     DONE;
1545   }
1546 )
1547
1548 ;; 0 is dst
1549 ;; 1 is src
1550 ;; 2 is size of copy in bytes
1551 ;; 3 is alignment
1552
1553 (define_expand "cpymemdi"
1554   [(match_operand:BLK 0 "memory_operand")
1555    (match_operand:BLK 1 "memory_operand")
1556    (match_operand:DI 2 "immediate_operand")
1557    (match_operand:DI 3 "immediate_operand")]
1558    "!STRICT_ALIGNMENT"
1559 {
1560   if (aarch64_expand_cpymem (operands))
1561     DONE;
1562   FAIL;
1563 }
1564 )
1565
1566 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1567 ;; fairly lax checking on the second memory operation.
1568 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1569   [(set (match_operand:SX 0 "register_operand" "=r,w")
1570         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1571    (set (match_operand:SX2 2 "register_operand" "=r,w")
1572         (match_operand:SX2 3 "memory_operand" "m,m"))]
1573    "rtx_equal_p (XEXP (operands[3], 0),
1574                  plus_constant (Pmode,
1575                                 XEXP (operands[1], 0),
1576                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1577   "@
1578    ldp\\t%w0, %w2, %z1
1579    ldp\\t%s0, %s2, %z1"
1580   [(set_attr "type" "load_8,neon_load1_2reg")
1581    (set_attr "arch" "*,fp")]
1582 )
1583
1584 ;; Storing different modes that can still be merged
1585 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1586   [(set (match_operand:DX 0 "register_operand" "=r,w")
1587         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1588    (set (match_operand:DX2 2 "register_operand" "=r,w")
1589         (match_operand:DX2 3 "memory_operand" "m,m"))]
1590    "rtx_equal_p (XEXP (operands[3], 0),
1591                  plus_constant (Pmode,
1592                                 XEXP (operands[1], 0),
1593                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1594   "@
1595    ldp\\t%x0, %x2, %z1
1596    ldp\\t%d0, %d2, %z1"
1597   [(set_attr "type" "load_16,neon_load1_2reg")
1598    (set_attr "arch" "*,fp")]
1599 )
1600
1601 (define_insn "load_pair_dw_tftf"
1602   [(set (match_operand:TF 0 "register_operand" "=w")
1603         (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1604    (set (match_operand:TF 2 "register_operand" "=w")
1605         (match_operand:TF 3 "memory_operand" "m"))]
1606    "TARGET_SIMD
1607     && rtx_equal_p (XEXP (operands[3], 0),
1608                     plus_constant (Pmode,
1609                                    XEXP (operands[1], 0),
1610                                    GET_MODE_SIZE (TFmode)))"
1611   "ldp\\t%q0, %q2, %z1"
1612   [(set_attr "type" "neon_ldp_q")
1613    (set_attr "fp" "yes")]
1614 )
1615
1616 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1617 ;; fairly lax checking on the second memory operation.
1618 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1619   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1620         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1621    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1622         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1623    "rtx_equal_p (XEXP (operands[2], 0),
1624                  plus_constant (Pmode,
1625                                 XEXP (operands[0], 0),
1626                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1627   "@
1628    stp\\t%w1, %w3, %z0
1629    stp\\t%s1, %s3, %z0"
1630   [(set_attr "type" "store_8,neon_store1_2reg")
1631    (set_attr "arch" "*,fp")]
1632 )
1633
1634 ;; Storing different modes that can still be merged
1635 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1636   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1637         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1638    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1639         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1640    "rtx_equal_p (XEXP (operands[2], 0),
1641                  plus_constant (Pmode,
1642                                 XEXP (operands[0], 0),
1643                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1644   "@
1645    stp\\t%x1, %x3, %z0
1646    stp\\t%d1, %d3, %z0"
1647   [(set_attr "type" "store_16,neon_store1_2reg")
1648    (set_attr "arch" "*,fp")]
1649 )
1650
1651 (define_insn "store_pair_dw_tftf"
1652   [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1653         (match_operand:TF 1 "register_operand" "w"))
1654    (set (match_operand:TF 2 "memory_operand" "=m")
1655         (match_operand:TF 3 "register_operand" "w"))]
1656    "TARGET_SIMD &&
1657     rtx_equal_p (XEXP (operands[2], 0),
1658                  plus_constant (Pmode,
1659                                 XEXP (operands[0], 0),
1660                                 GET_MODE_SIZE (TFmode)))"
1661   "stp\\t%q1, %q3, %z0"
1662   [(set_attr "type" "neon_stp_q")
1663    (set_attr "fp" "yes")]
1664 )
1665
1666 ;; Load pair with post-index writeback.  This is primarily used in function
1667 ;; epilogues.
1668 ;;
1669 ;; MORELLO TODO: pure-cap.
1670 (define_insn "@loadwb_pair<GPI:mode>_<ADDR:mode>"
1671   [(parallel
1672     [(set (match_operand:ADDR 0 "register_operand" "=k")
1673           (<ADDR:PLUS>:ADDR
1674             (match_operand:ADDR 1 "register_operand" "0")
1675             (match_operand:DI 4 "aarch64_mem_pair_offset" "n")))
1676      (set (match_operand:GPI 2 "register_operand" "=r")
1677           (mem:GPI (match_dup 1)))
1678      (set (match_operand:GPI 3 "register_operand" "=r")
1679           (mem:GPI (<ADDR:PLUS>:ADDR
1680                      (match_dup 1)
1681                      (match_operand:DI 5 "const_int_operand" "n"))))])]
1682   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1683   "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1684   [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1685 )
1686
1687 (define_insn "@loadwb_pair<GPF:mode>_<ADDR:mode>"
1688   [(parallel
1689     [(set (match_operand:ADDR 0 "register_operand" "=k")
1690           (<ADDR:PLUS>:ADDR
1691             (match_operand:ADDR 1 "register_operand" "0")
1692             (match_operand:DI 4 "aarch64_mem_pair_offset" "n")))
1693      (set (match_operand:GPF 2 "register_operand" "=w")
1694           (mem:GPF (match_dup 1)))
1695      (set (match_operand:GPF 3 "register_operand" "=w")
1696           (mem:GPF (<ADDR:PLUS>:ADDR
1697                      (match_dup 1)
1698                      (match_operand:DI 5 "const_int_operand" "n"))))])]
1699   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1700   "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1701   [(set_attr "type" "neon_load1_2reg")]
1702 )
1703
1704 (define_insn "@loadwb_pair<TX:mode>_<ADDR:mode>"
1705   [(parallel
1706     [(set (match_operand:ADDR 0 "register_operand" "=k")
1707           (<ADDR:PLUS>:ADDR
1708             (match_operand:ADDR 1 "register_operand" "0")
1709             (match_operand:DI 4 "aarch64_mem_pair_offset" "n")))
1710      (set (match_operand:TX 2 "register_operand" "=w")
1711           (mem:TX (match_dup 1)))
1712      (set (match_operand:TX 3 "register_operand" "=w")
1713           (mem:TX (<ADDR:PLUS>:ADDR
1714                     (match_dup 1)
1715                     (match_operand:DI 5 "const_int_operand" "n"))))])]
1716   "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1717   "ldp\\t%q2, %q3, [%1], %4"
1718   [(set_attr "type" "neon_ldp_q")]
1719 )
1720
1721 ;; Store pair with pre-index writeback.  This is primarily used in function
1722 ;; prologues.
1723 ;;
1724 ;; MORELLO TODO: pure-cap.
1725 (define_insn "@storewb_pair<GPI:mode>_<ADDR:mode>"
1726   [(parallel
1727     [(set (match_operand:ADDR 0 "register_operand" "=&k")
1728           (<ADDR:PLUS>:ADDR
1729             (match_operand:ADDR 1 "register_operand" "0")
1730             (match_operand:DI 4 "aarch64_mem_pair_offset" "n")))
1731      (set (mem:GPI (<ADDR:PLUS>:ADDR (match_dup 0) (match_dup 4)))
1732           (match_operand:GPI 2 "register_operand" "r"))
1733      (set (mem:GPI (<ADDR:PLUS>:ADDR
1734                      (match_dup 0)
1735                      (match_operand:DI 5 "const_int_operand" "n")))
1736           (match_operand:GPI 3 "register_operand" "r"))])]
1737   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1738   "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1739   [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1740 )
1741
1742 (define_insn "@storewb_pair<GPF:mode>_<ADDR:mode>"
1743   [(parallel
1744     [(set (match_operand:ADDR 0 "register_operand" "=&k")
1745           (<ADDR:PLUS>:ADDR
1746             (match_operand:ADDR 1 "register_operand" "0")
1747             (match_operand:DI 4 "aarch64_mem_pair_offset" "n")))
1748      (set (mem:GPF (<ADDR:PLUS>:ADDR (match_dup 0) (match_dup 4)))
1749           (match_operand:GPF 2 "register_operand" "w"))
1750      (set (mem:GPF (<ADDR:PLUS>:ADDR
1751                      (match_dup 0)
1752                      (match_operand:DI 5 "const_int_operand" "n")))
1753           (match_operand:GPF 3 "register_operand" "w"))])]
1754   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1755   "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1756   [(set_attr "type" "neon_store1_2reg<q>")]
1757 )
1758
1759 (define_insn "@storewb_pair<TX:mode>_<ADDR:mode>"
1760   [(parallel
1761     [(set (match_operand:ADDR 0 "register_operand" "=&k")
1762           (<ADDR:PLUS>:ADDR
1763             (match_operand:ADDR 1 "register_operand" "0")
1764             (match_operand:DI 4 "aarch64_mem_pair_offset" "n")))
1765      (set (mem:TX (<ADDR:PLUS>:ADDR (match_dup 0) (match_dup 4)))
1766           (match_operand:TX 2 "register_operand" "w"))
1767      (set (mem:TX (<ADDR:PLUS>:ADDR
1768                     (match_dup 0)
1769                     (match_operand:DI 5 "const_int_operand" "n")))
1770           (match_operand:TX 3 "register_operand" "w"))])]
1771   "TARGET_SIMD
1772    && INTVAL (operands[5])
1773       == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1774   "stp\\t%q2, %q3, [%0, %4]!"
1775   [(set_attr "type" "neon_stp_q")]
1776 )
1777
1778 ;; -------------------------------------------------------------------
1779 ;; Sign/Zero extension
1780 ;; -------------------------------------------------------------------
1781
1782 (define_expand "<optab>sidi2"
1783   [(set (match_operand:DI 0 "register_operand")
1784         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1785   ""
1786 )
1787
1788 (define_insn "*extendsidi2_aarch64"
1789   [(set (match_operand:DI 0 "register_operand" "=r,r")
1790         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1791   ""
1792   "@
1793    sxtw\t%0, %w1
1794    ldrsw\t%0, %1"
1795   [(set_attr "type" "extend,load_4")]
1796 )
1797
1798 (define_insn "*load_pair_extendsidi2_aarch64"
1799   [(set (match_operand:DI 0 "register_operand" "=r")
1800         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1801    (set (match_operand:DI 2 "register_operand" "=r")
1802         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1803   "rtx_equal_p (XEXP (operands[3], 0),
1804                 plus_constant (Pmode,
1805                                XEXP (operands[1], 0),
1806                                GET_MODE_SIZE (SImode)))"
1807   "ldpsw\\t%0, %2, %z1"
1808   [(set_attr "type" "load_8")]
1809 )
1810
1811 (define_insn "*zero_extendsidi2_aarch64"
1812   [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1813         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1814   ""
1815   "@
1816    uxtw\t%0, %w1
1817    ldr\t%w0, %1
1818    fmov\t%s0, %w1
1819    ldr\t%s0, %1
1820    fmov\t%w0, %s1
1821    fmov\t%s0, %s1"
1822   [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1823    (set_attr "arch" "*,*,fp,fp,fp,fp")]
1824 )
1825
1826 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1827   [(set (match_operand:DI 0 "register_operand" "=r,w")
1828         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1829    (set (match_operand:DI 2 "register_operand" "=r,w")
1830         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1831   "rtx_equal_p (XEXP (operands[3], 0),
1832                 plus_constant (Pmode,
1833                                XEXP (operands[1], 0),
1834                                GET_MODE_SIZE (SImode)))"
1835   "@
1836    ldp\t%w0, %w2, %z1
1837    ldp\t%s0, %s2, %z1"
1838   [(set_attr "type" "load_8,neon_load1_2reg")
1839    (set_attr "arch" "*,fp")]
1840 )
1841
1842 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1843   [(set (match_operand:GPI 0 "register_operand")
1844         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1845   ""
1846 )
1847
1848 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1849   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1850         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1851   ""
1852   "@
1853    sxt<SHORT:size>\t%<GPI:w>0, %w1
1854    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1855   [(set_attr "type" "extend,load_4")]
1856 )
1857
1858 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1859   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1860         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1861   ""
1862   "@
1863    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1864    ldr<SHORT:size>\t%w0, %1
1865    ldr\t%<SHORT:size>0, %1"
1866   [(set_attr "type" "logic_imm,load_4,f_loads")
1867    (set_attr "arch" "*,*,fp")]
1868 )
1869
1870 (define_expand "<optab>qihi2"
1871   [(set (match_operand:HI 0 "register_operand")
1872         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1873   ""
1874 )
1875
1876 (define_insn "*extendqihi2_aarch64"
1877   [(set (match_operand:HI 0 "register_operand" "=r,r")
1878         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1879   ""
1880   "@
1881    sxtb\t%w0, %w1
1882    ldrsb\t%w0, %1"
1883   [(set_attr "type" "extend,load_4")]
1884 )
1885
1886 (define_insn "*zero_extendqihi2_aarch64"
1887   [(set (match_operand:HI 0 "register_operand" "=r,r")
1888         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1889   ""
1890   "@
1891    and\t%w0, %w1, 255
1892    ldrb\t%w0, %1"
1893   [(set_attr "type" "logic_imm,load_4")]
1894 )
1895
1896 ;; -------------------------------------------------------------------
1897 ;; Simple arithmetic
1898 ;; -------------------------------------------------------------------
1899
1900 (define_expand "add<mode>3"
1901   [(set
1902     (match_operand:GPI 0 "register_operand")
1903     (plus:GPI (match_operand:GPI 1 "register_operand")
1904               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1905   ""
1906 {
1907   /* If operands[1] is a subreg extract the inner RTX.  */
1908   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1909
1910   /* If the constant is too large for a single instruction and isn't frame
1911      based, split off the immediate so it is available for CSE.  */
1912   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1913       && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
1914       && can_create_pseudo_p ()
1915       && (!REG_P (op1)
1916          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1917     operands[2] = force_reg (<MODE>mode, operands[2]);
1918   /* Expand polynomial additions now if the destination is the stack
1919      pointer, since we don't want to use that as a temporary.  */
1920   else if (operands[0] == stack_pointer_rtx
1921            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1922     {
1923       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1924                                 operands[2], NULL_RTX, NULL_RTX);
1925       DONE;
1926     }
1927 })
1928
1929 (define_insn "*add<mode>3_aarch64"
1930   [(set
1931     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
1932     (plus:GPI
1933      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1934      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
1935   ""
1936   "@
1937   add\\t%<w>0, %<w>1, %2
1938   add\\t%<w>0, %<w>1, %<w>2
1939   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1940   sub\\t%<w>0, %<w>1, #%n2
1941   #
1942   * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1943   * return aarch64_output_sve_addvl_addpl (operands[2]);"
1944   ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1945   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1946    (set_attr "arch" "*,*,simd,*,*,sve,sve")]
1947 )
1948
1949 ;; zero_extend version of above
1950 (define_insn "*addsi3_aarch64_uxtw"
1951   [(set
1952     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1953     (zero_extend:DI
1954      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1955               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1956   ""
1957   "@
1958   add\\t%w0, %w1, %2
1959   add\\t%w0, %w1, %w2
1960   sub\\t%w0, %w1, #%n2
1961   #"
1962   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1963 )
1964
1965 ;; If there's a free register, and we can load the constant with a
1966 ;; single instruction, do so.  This has a chance to improve scheduling.
1967 (define_peephole2
1968   [(match_scratch:GPI 3 "r")
1969    (set (match_operand:GPI 0 "register_operand")
1970         (plus:GPI
1971           (match_operand:GPI 1 "register_operand")
1972           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1973   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1974   [(set (match_dup 3) (match_dup 2))
1975    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1976 )
1977
1978 (define_peephole2
1979   [(match_scratch:SI 3 "r")
1980    (set (match_operand:DI 0 "register_operand")
1981         (zero_extend:DI
1982           (plus:SI
1983             (match_operand:SI 1 "register_operand")
1984             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1985   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1986   [(set (match_dup 3) (match_dup 2))
1987    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1988 )
1989
1990 ;; After peephole2 has had a chance to run, split any remaining long
1991 ;; additions into two add immediates.
1992 (define_split
1993   [(set (match_operand:GPI 0 "register_operand")
1994         (plus:GPI
1995           (match_operand:GPI 1 "register_operand")
1996           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1997   "epilogue_completed"
1998   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1999    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
2000   {
2001     HOST_WIDE_INT i = INTVAL (operands[2]);
2002     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2003     operands[3] = GEN_INT (i - s);
2004     operands[4] = GEN_INT (s);
2005   }
2006 )
2007
2008 ;; Match addition of polynomial offsets that require one temporary, for which
2009 ;; we can use the early-clobbered destination register.  This is a separate
2010 ;; pattern so that the early clobber doesn't affect register allocation
2011 ;; for other forms of addition.  However, we still need to provide an
2012 ;; all-register alternative, in case the offset goes out of range after
2013 ;; elimination.  For completeness we might as well provide all GPR-based
2014 ;; alternatives from the main pattern.
2015 ;;
2016 ;; We don't have a pattern for additions requiring two temporaries since at
2017 ;; present LRA doesn't allow new scratches to be added during elimination.
2018 ;; Such offsets should be rare anyway.
2019 ;;
2020 ;; ??? But if we added LRA support for new scratches, much of the ugliness
2021 ;; here would go away.  We could just handle all polynomial constants in
2022 ;; this pattern.
2023 (define_insn_and_split "*add<mode>3_poly_1"
2024   [(set
2025     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
2026     (plus:GPI
2027      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
2028      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
2029   "TARGET_SVE && operands[0] != stack_pointer_rtx"
2030   "@
2031   add\\t%<w>0, %<w>1, %2
2032   add\\t%<w>0, %<w>1, %<w>2
2033   sub\\t%<w>0, %<w>1, #%n2
2034   #
2035   * return aarch64_output_sve_scalar_inc_dec (operands[2]);
2036   * return aarch64_output_sve_addvl_addpl (operands[2]);
2037   #"
2038   "&& epilogue_completed
2039    && !reg_overlap_mentioned_p (operands[0], operands[1])
2040    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
2041   [(const_int 0)]
2042   {
2043     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
2044                               operands[2], operands[0], NULL_RTX);
2045     DONE;
2046   }
2047   ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
2048   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
2049 )
2050
2051 (define_split
2052   [(set (match_operand:DI 0 "register_operand")
2053         (zero_extend:DI
2054           (plus:SI
2055             (match_operand:SI 1 "register_operand")
2056             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
2057   "epilogue_completed"
2058   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
2059    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
2060   {
2061     HOST_WIDE_INT i = INTVAL (operands[2]);
2062     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2063     operands[3] = GEN_INT (i - s);
2064     operands[4] = GEN_INT (s);
2065     operands[5] = gen_lowpart (SImode, operands[0]);
2066   }
2067 )
2068
2069 (define_expand "addv<mode>4"
2070   [(match_operand:GPI 0 "register_operand")
2071    (match_operand:GPI 1 "register_operand")
2072    (match_operand:GPI 2 "aarch64_plus_operand")
2073    (label_ref (match_operand 3 "" ""))]
2074   ""
2075 {
2076   if (CONST_INT_P (operands[2]))
2077     emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
2078                                             operands[2]));
2079   else
2080     emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
2081   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2082
2083   DONE;
2084 })
2085
2086 (define_expand "uaddv<mode>4"
2087   [(match_operand:GPI 0 "register_operand")
2088    (match_operand:GPI 1 "register_operand")
2089    (match_operand:GPI 2 "register_operand")
2090    (label_ref (match_operand 3 "" ""))]
2091   ""
2092 {
2093   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
2094   aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
2095
2096   DONE;
2097 })
2098
2099 (define_expand "addti3"
2100   [(set (match_operand:TI 0 "register_operand")
2101         (plus:TI (match_operand:TI 1 "register_operand")
2102                  (match_operand:TI 2 "aarch64_reg_or_imm")))]
2103   ""
2104 {
2105   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2106
2107   aarch64_addti_scratch_regs (operands[1], operands[2],
2108                               &low_dest, &op1_low, &op2_low,
2109                               &high_dest, &op1_high, &op2_high);
2110
2111   if (op2_low == const0_rtx)
2112     {
2113       low_dest = op1_low;
2114       if (!aarch64_pluslong_operand (op2_high, DImode))
2115         op2_high = force_reg (DImode, op2_high);
2116       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
2117     }
2118   else
2119     {
2120       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2121                                       force_reg (DImode, op2_low)));
2122       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
2123                                      force_reg (DImode, op2_high)));
2124     }
2125
2126   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2127   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2128
2129   DONE;
2130 })
2131
2132 (define_expand "addvti4"
2133   [(match_operand:TI 0 "register_operand")
2134    (match_operand:TI 1 "register_operand")
2135    (match_operand:TI 2 "aarch64_reg_or_imm")
2136    (label_ref (match_operand 3 "" ""))]
2137   ""
2138 {
2139   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2140
2141   aarch64_addti_scratch_regs (operands[1], operands[2],
2142                               &low_dest, &op1_low, &op2_low,
2143                               &high_dest, &op1_high, &op2_high);
2144
2145   if (op2_low == const0_rtx)
2146     {
2147       low_dest = op1_low;
2148       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2149                                       force_reg (DImode, op2_high)));
2150     }
2151   else
2152     {
2153       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2154                                       force_reg (DImode, op2_low)));
2155       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2156                                       force_reg (DImode, op2_high)));
2157     }
2158
2159   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2160   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2161
2162   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2163   DONE;
2164 })
2165
2166 (define_expand "uaddvti4"
2167   [(match_operand:TI 0 "register_operand")
2168    (match_operand:TI 1 "register_operand")
2169    (match_operand:TI 2 "aarch64_reg_or_imm")
2170    (label_ref (match_operand 3 "" ""))]
2171   ""
2172 {
2173   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2174
2175   aarch64_addti_scratch_regs (operands[1], operands[2],
2176                               &low_dest, &op1_low, &op2_low,
2177                               &high_dest, &op1_high, &op2_high);
2178
2179   if (op2_low == const0_rtx)
2180     {
2181       low_dest = op1_low;
2182       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2183                                       force_reg (DImode, op2_high)));
2184     }
2185   else
2186     {
2187       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2188                                       force_reg (DImode, op2_low)));
2189       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2190                                       force_reg (DImode, op2_high)));
2191     }
2192
2193   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2194   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2195
2196   aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2197   DONE;
2198  })
2199
2200 (define_insn "add<mode>3_compare0"
2201   [(set (reg:CC_NZ CC_REGNUM)
2202         (compare:CC_NZ
2203          (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2204                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2205          (const_int 0)))
2206    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2207         (plus:GPI (match_dup 1) (match_dup 2)))]
2208   ""
2209   "@
2210   adds\\t%<w>0, %<w>1, %<w>2
2211   adds\\t%<w>0, %<w>1, %2
2212   subs\\t%<w>0, %<w>1, #%n2"
2213   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2214 )
2215
2216 ;; zero_extend version of above
2217 (define_insn "*addsi3_compare0_uxtw"
2218   [(set (reg:CC_NZ CC_REGNUM)
2219         (compare:CC_NZ
2220          (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2221                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2222          (const_int 0)))
2223    (set (match_operand:DI 0 "register_operand" "=r,r,r")
2224         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2225   ""
2226   "@
2227   adds\\t%w0, %w1, %w2
2228   adds\\t%w0, %w1, %2
2229   subs\\t%w0, %w1, #%n2"
2230   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2231 )
2232
2233 (define_insn "*add<mode>3_compareC_cconly"
2234   [(set (reg:CC_C CC_REGNUM)
2235         (compare:CC_C
2236           (plus:GPI
2237             (match_operand:GPI 0 "register_operand" "r,r,r")
2238             (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2239           (match_dup 0)))]
2240   ""
2241   "@
2242   cmn\\t%<w>0, %<w>1
2243   cmn\\t%<w>0, %1
2244   cmp\\t%<w>0, #%n1"
2245   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2246 )
2247
2248 (define_insn "add<mode>3_compareC"
2249   [(set (reg:CC_C CC_REGNUM)
2250         (compare:CC_C
2251           (plus:GPI
2252             (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2253             (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2254           (match_dup 1)))
2255    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2256         (plus:GPI (match_dup 1) (match_dup 2)))]
2257   ""
2258   "@
2259   adds\\t%<w>0, %<w>1, %<w>2
2260   adds\\t%<w>0, %<w>1, %2
2261   subs\\t%<w>0, %<w>1, #%n2"
2262   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2263 )
2264
2265 (define_insn "*add<mode>3_compareV_cconly_imm"
2266   [(set (reg:CC_V CC_REGNUM)
2267         (compare:CC_V
2268           (plus:<DWI>
2269             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2270             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2271           (sign_extend:<DWI>
2272            (plus:GPI
2273             (match_dup 0)
2274             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2275   "INTVAL (operands[1]) == INTVAL (operands[2])"
2276   "@
2277   cmn\\t%<w>0, %<w>1
2278   cmp\\t%<w>0, #%n1"
2279   [(set_attr "type" "alus_imm")]
2280 )
2281
2282 (define_insn "*add<mode>3_compareV_cconly"
2283   [(set (reg:CC_V CC_REGNUM)
2284         (compare:CC_V
2285           (plus:<DWI>
2286             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2287             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2288           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2289   ""
2290   "cmn\\t%<w>0, %<w>1"
2291   [(set_attr "type" "alus_sreg")]
2292 )
2293
2294 (define_insn "add<mode>3_compareV_imm"
2295   [(set (reg:CC_V CC_REGNUM)
2296         (compare:CC_V
2297           (plus:<DWI>
2298             (sign_extend:<DWI>
2299               (match_operand:GPI 1 "register_operand" "rk,rk"))
2300             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2301           (sign_extend:<DWI>
2302             (plus:GPI (match_dup 1) (match_dup 2)))))
2303    (set (match_operand:GPI 0 "register_operand" "=r,r")
2304         (plus:GPI (match_dup 1) (match_dup 2)))]
2305    ""
2306    "@
2307    adds\\t%<w>0, %<w>1, %<w>2
2308    subs\\t%<w>0, %<w>1, #%n2"
2309   [(set_attr "type" "alus_imm,alus_imm")]
2310 )
2311
2312 (define_insn "add<mode>3_compareV"
2313   [(set (reg:CC_V CC_REGNUM)
2314         (compare:CC_V
2315           (plus:<DWI>
2316             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2317             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2318           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2319    (set (match_operand:GPI 0 "register_operand" "=r")
2320         (plus:GPI (match_dup 1) (match_dup 2)))]
2321   ""
2322   "adds\\t%<w>0, %<w>1, %<w>2"
2323   [(set_attr "type" "alus_sreg")]
2324 )
2325
2326 (define_insn "*adds_shift_imm_<mode>"
2327   [(set (reg:CC_NZ CC_REGNUM)
2328         (compare:CC_NZ
2329          (plus:GPI (ASHIFT:GPI
2330                     (match_operand:GPI 1 "register_operand" "r")
2331                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2332                    (match_operand:GPI 3 "register_operand" "r"))
2333          (const_int 0)))
2334    (set (match_operand:GPI 0 "register_operand" "=r")
2335         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2336                   (match_dup 3)))]
2337   ""
2338   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2339   [(set_attr "type" "alus_shift_imm")]
2340 )
2341
2342 (define_insn "*subs_shift_imm_<mode>"
2343   [(set (reg:CC_NZ CC_REGNUM)
2344         (compare:CC_NZ
2345          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2346                     (ASHIFT:GPI
2347                      (match_operand:GPI 2 "register_operand" "r")
2348                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2349          (const_int 0)))
2350    (set (match_operand:GPI 0 "register_operand" "=r")
2351         (minus:GPI (match_dup 1)
2352                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2353   ""
2354   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2355   [(set_attr "type" "alus_shift_imm")]
2356 )
2357
2358 (define_insn "*adds_mul_imm_<mode>"
2359   [(set (reg:CC_NZ CC_REGNUM)
2360         (compare:CC_NZ
2361          (plus:GPI (mult:GPI
2362                     (match_operand:GPI 1 "register_operand" "r")
2363                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2364                    (match_operand:GPI 3 "register_operand" "r"))
2365          (const_int 0)))
2366    (set (match_operand:GPI 0 "register_operand" "=r")
2367         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2368                   (match_dup 3)))]
2369   ""
2370   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2371   [(set_attr "type" "alus_shift_imm")]
2372 )
2373
2374 (define_insn "*subs_mul_imm_<mode>"
2375   [(set (reg:CC_NZ CC_REGNUM)
2376         (compare:CC_NZ
2377          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2378                     (mult:GPI
2379                      (match_operand:GPI 2 "register_operand" "r")
2380                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2381          (const_int 0)))
2382    (set (match_operand:GPI 0 "register_operand" "=r")
2383         (minus:GPI (match_dup 1)
2384                    (mult:GPI (match_dup 2) (match_dup 3))))]
2385   ""
2386   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2387   [(set_attr "type" "alus_shift_imm")]
2388 )
2389
2390 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2391   [(set (reg:CC_NZ CC_REGNUM)
2392         (compare:CC_NZ
2393          (plus:GPI
2394           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2395           (match_operand:GPI 2 "register_operand" "rk"))
2396         (const_int 0)))
2397    (set (match_operand:GPI 0 "register_operand" "=r")
2398         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2399   ""
2400   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2401   [(set_attr "type" "alus_ext")]
2402 )
2403
2404 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2405   [(set (reg:CC_NZ CC_REGNUM)
2406         (compare:CC_NZ
2407          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2408                     (ANY_EXTEND:GPI
2409                      (match_operand:ALLX 2 "register_operand" "r")))
2410         (const_int 0)))
2411    (set (match_operand:GPI 0 "register_operand" "=r")
2412         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2413   ""
2414   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2415   [(set_attr "type" "alus_ext")]
2416 )
2417
2418 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2419   [(set (reg:CC_NZ CC_REGNUM)
2420         (compare:CC_NZ
2421          (plus:GPI (ashift:GPI
2422                     (ANY_EXTEND:GPI
2423                      (match_operand:ALLX 1 "register_operand" "r"))
2424                     (match_operand 2 "aarch64_imm3" "Ui3"))
2425                    (match_operand:GPI 3 "register_operand" "rk"))
2426          (const_int 0)))
2427    (set (match_operand:GPI 0 "register_operand" "=rk")
2428         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2429                               (match_dup 2))
2430                   (match_dup 3)))]
2431   ""
2432   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2433   [(set_attr "type" "alus_ext")]
2434 )
2435
2436 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2437   [(set (reg:CC_NZ CC_REGNUM)
2438         (compare:CC_NZ
2439          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2440                     (ashift:GPI
2441                      (ANY_EXTEND:GPI
2442                       (match_operand:ALLX 2 "register_operand" "r"))
2443                      (match_operand 3 "aarch64_imm3" "Ui3")))
2444          (const_int 0)))
2445    (set (match_operand:GPI 0 "register_operand" "=rk")
2446         (minus:GPI (match_dup 1)
2447                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2448                                (match_dup 3))))]
2449   ""
2450   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2451   [(set_attr "type" "alus_ext")]
2452 )
2453
2454 (define_insn "*adds_<optab><mode>_multp2"
2455   [(set (reg:CC_NZ CC_REGNUM)
2456         (compare:CC_NZ
2457          (plus:GPI (ANY_EXTRACT:GPI
2458                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2459                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2460                     (match_operand 3 "const_int_operand" "n")
2461                     (const_int 0))
2462                    (match_operand:GPI 4 "register_operand" "rk"))
2463         (const_int 0)))
2464    (set (match_operand:GPI 0 "register_operand" "=r")
2465         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2466                                    (match_dup 3)
2467                                    (const_int 0))
2468                   (match_dup 4)))]
2469   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2470   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2471   [(set_attr "type" "alus_ext")]
2472 )
2473
2474 (define_insn "*subs_<optab><mode>_multp2"
2475   [(set (reg:CC_NZ CC_REGNUM)
2476         (compare:CC_NZ
2477          (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2478                     (ANY_EXTRACT:GPI
2479                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2480                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2481                      (match_operand 3 "const_int_operand" "n")
2482                      (const_int 0)))
2483         (const_int 0)))
2484    (set (match_operand:GPI 0 "register_operand" "=r")
2485         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2486                                   (mult:GPI (match_dup 1) (match_dup 2))
2487                                   (match_dup 3)
2488                                   (const_int 0))))]
2489   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2490   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2491   [(set_attr "type" "alus_ext")]
2492 )
2493
2494 (define_insn "*add<mode>3nr_compare0"
2495   [(set (reg:CC_NZ CC_REGNUM)
2496         (compare:CC_NZ
2497          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2498                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2499          (const_int 0)))]
2500   ""
2501   "@
2502   cmn\\t%<w>0, %<w>1
2503   cmn\\t%<w>0, %1
2504   cmp\\t%<w>0, #%n1"
2505   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2506 )
2507
2508 (define_insn "aarch64_sub<mode>_compare0"
2509   [(set (reg:CC_NZ CC_REGNUM)
2510         (compare:CC_NZ
2511          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2512                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2513          (const_int 0)))]
2514   ""
2515   "cmp\\t%<w>0, %<w>1"
2516   [(set_attr "type" "alus_sreg")]
2517 )
2518
2519 (define_insn "*compare_neg<mode>"
2520   [(set (reg:CC_Z CC_REGNUM)
2521         (compare:CC_Z
2522          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2523          (match_operand:GPI 1 "register_operand" "r")))]
2524   ""
2525   "cmn\\t%<w>1, %<w>0"
2526   [(set_attr "type" "alus_sreg")]
2527 )
2528
2529 (define_insn "*add_<shift>_<mode>"
2530   [(set (match_operand:GPI 0 "register_operand" "=r")
2531         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2532                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2533                   (match_operand:GPI 3 "register_operand" "r")))]
2534   ""
2535   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2536   [(set_attr "type" "alu_shift_imm")]
2537 )
2538
2539 ;; zero_extend version of above
2540 (define_insn "*add_<shift>_si_uxtw"
2541   [(set (match_operand:DI 0 "register_operand" "=r")
2542         (zero_extend:DI
2543          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2544                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2545                   (match_operand:SI 3 "register_operand" "r"))))]
2546   ""
2547   "add\\t%w0, %w3, %w1, <shift> %2"
2548   [(set_attr "type" "alu_shift_imm")]
2549 )
2550
2551 (define_insn "*add_mul_imm_<mode>"
2552   [(set (match_operand:GPI 0 "register_operand" "=r")
2553         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2554                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2555                   (match_operand:GPI 3 "register_operand" "r")))]
2556   ""
2557   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2558   [(set_attr "type" "alu_shift_imm")]
2559 )
2560
2561 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2562   [(set (match_operand:GPI 0 "register_operand" "=rk")
2563         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2564                   (match_operand:GPI 2 "register_operand" "r")))]
2565   ""
2566   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2567   [(set_attr "type" "alu_ext")]
2568 )
2569
2570 ;; zero_extend version of above
2571 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2572   [(set (match_operand:DI 0 "register_operand" "=rk")
2573         (zero_extend:DI
2574          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2575                   (match_operand:GPI 2 "register_operand" "r"))))]
2576   ""
2577   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2578   [(set_attr "type" "alu_ext")]
2579 )
2580
2581 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2582   [(set (match_operand:GPI 0 "register_operand" "=rk")
2583         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2584                                (match_operand:ALLX 1 "register_operand" "r"))
2585                               (match_operand 2 "aarch64_imm3" "Ui3"))
2586                   (match_operand:GPI 3 "register_operand" "r")))]
2587   ""
2588   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2589   [(set_attr "type" "alu_ext")]
2590 )
2591
2592 ;; zero_extend version of above
2593 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2594   [(set (match_operand:DI 0 "register_operand" "=rk")
2595         (zero_extend:DI
2596          (plus:SI (ashift:SI (ANY_EXTEND:SI
2597                               (match_operand:SHORT 1 "register_operand" "r"))
2598                              (match_operand 2 "aarch64_imm3" "Ui3"))
2599                   (match_operand:SI 3 "register_operand" "r"))))]
2600   ""
2601   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2602   [(set_attr "type" "alu_ext")]
2603 )
2604
2605 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2606   [(set (match_operand:GPI 0 "register_operand" "=rk")
2607         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2608                              (match_operand:ALLX 1 "register_operand" "r"))
2609                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2610                   (match_operand:GPI 3 "register_operand" "r")))]
2611   ""
2612   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2613   [(set_attr "type" "alu_ext")]
2614 )
2615
2616 ;; zero_extend version of above
2617 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2618   [(set (match_operand:DI 0 "register_operand" "=rk")
2619         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2620                              (match_operand:SHORT 1 "register_operand" "r"))
2621                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2622                   (match_operand:SI 3 "register_operand" "r"))))]
2623   ""
2624   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2625   [(set_attr "type" "alu_ext")]
2626 )
2627
2628 (define_insn "*add_<optab><mode>_multp2"
2629   [(set (match_operand:GPI 0 "register_operand" "=rk")
2630         (plus:GPI (ANY_EXTRACT:GPI
2631                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2632                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2633                    (match_operand 3 "const_int_operand" "n")
2634                    (const_int 0))
2635                   (match_operand:GPI 4 "register_operand" "r")))]
2636   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2637   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2638   [(set_attr "type" "alu_ext")]
2639 )
2640
2641 ;; zero_extend version of above
2642 (define_insn "*add_<optab>si_multp2_uxtw"
2643   [(set (match_operand:DI 0 "register_operand" "=rk")
2644         (zero_extend:DI
2645          (plus:SI (ANY_EXTRACT:SI
2646                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2647                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2648                    (match_operand 3 "const_int_operand" "n")
2649                    (const_int 0))
2650                   (match_operand:SI 4 "register_operand" "r"))))]
2651   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2652   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2653   [(set_attr "type" "alu_ext")]
2654 )
2655
2656 (define_expand "add<mode>3_carryin"
2657   [(set (match_operand:GPI 0 "register_operand")
2658         (plus:GPI
2659           (plus:GPI
2660             (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2661             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2662           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2663    ""
2664    ""
2665 )
2666
2667 ;; Note that add with carry with two zero inputs is matched by cset,
2668 ;; and that add with carry with one zero input is matched by cinc.
2669
2670 (define_insn "*add<mode>3_carryin"
2671   [(set (match_operand:GPI 0 "register_operand" "=r")
2672         (plus:GPI
2673           (plus:GPI
2674             (match_operand:GPI 3 "aarch64_carry_operation" "")
2675             (match_operand:GPI 1 "register_operand" "r"))
2676           (match_operand:GPI 2 "register_operand" "r")))]
2677    ""
2678    "adc\\t%<w>0, %<w>1, %<w>2"
2679   [(set_attr "type" "adc_reg")]
2680 )
2681
2682 ;; zero_extend version of above
2683 (define_insn "*addsi3_carryin_uxtw"
2684   [(set (match_operand:DI 0 "register_operand" "=r")
2685         (zero_extend:DI
2686           (plus:SI
2687             (plus:SI
2688               (match_operand:SI 3 "aarch64_carry_operation" "")
2689               (match_operand:SI 1 "register_operand" "r"))
2690             (match_operand:SI 2 "register_operand" "r"))))]
2691    ""
2692    "adc\\t%w0, %w1, %w2"
2693   [(set_attr "type" "adc_reg")]
2694 )
2695
2696 (define_expand "add<mode>3_carryinC"
2697   [(parallel
2698      [(set (match_dup 3)
2699            (compare:CC_ADC
2700              (plus:<DWI>
2701                (plus:<DWI>
2702                  (match_dup 4)
2703                  (zero_extend:<DWI>
2704                    (match_operand:GPI 1 "register_operand")))
2705                (zero_extend:<DWI>
2706                  (match_operand:GPI 2 "register_operand")))
2707              (match_dup 6)))
2708       (set (match_operand:GPI 0 "register_operand")
2709            (plus:GPI
2710              (plus:GPI (match_dup 5) (match_dup 1))
2711              (match_dup 2)))])]
2712    ""
2713 {
2714   operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2715   rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2716   operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2717   operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2718   operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2719                                       << GET_MODE_BITSIZE (<MODE>mode),
2720                                       TImode);
2721 })
2722
2723 (define_insn "*add<mode>3_carryinC_zero"
2724   [(set (reg:CC_ADC CC_REGNUM)
2725         (compare:CC_ADC
2726           (plus:<DWI>
2727             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2728             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2729           (match_operand 4 "const_scalar_int_operand" "")))
2730    (set (match_operand:GPI 0 "register_operand" "=r")
2731         (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2732                   (match_dup 1)))]
2733   "rtx_mode_t (operands[4], <DWI>mode)
2734    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2735    "adcs\\t%<w>0, %<w>1, <w>zr"
2736   [(set_attr "type" "adc_reg")]
2737 )
2738
2739 (define_insn "*add<mode>3_carryinC"
2740   [(set (reg:CC_ADC CC_REGNUM)
2741         (compare:CC_ADC
2742           (plus:<DWI>
2743             (plus:<DWI>
2744               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2745               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2746             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2747           (match_operand 5 "const_scalar_int_operand" "")))
2748    (set (match_operand:GPI 0 "register_operand" "=r")
2749         (plus:GPI
2750           (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2751                     (match_dup 1))
2752           (match_dup 2)))]
2753   "rtx_mode_t (operands[5], <DWI>mode)
2754    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2755    "adcs\\t%<w>0, %<w>1, %<w>2"
2756   [(set_attr "type" "adc_reg")]
2757 )
2758
2759 (define_expand "add<mode>3_carryinV"
2760   [(parallel
2761      [(set (reg:CC_V CC_REGNUM)
2762            (compare:CC_V
2763              (plus:<DWI>
2764                (plus:<DWI>
2765                  (match_dup 3)
2766                  (sign_extend:<DWI>
2767                    (match_operand:GPI 1 "register_operand")))
2768                (sign_extend:<DWI>
2769                  (match_operand:GPI 2 "register_operand")))
2770            (sign_extend:<DWI>
2771              (plus:GPI
2772                (plus:GPI (match_dup 4) (match_dup 1))
2773                (match_dup 2)))))
2774       (set (match_operand:GPI 0 "register_operand")
2775            (plus:GPI
2776              (plus:GPI (match_dup 4) (match_dup 1))
2777              (match_dup 2)))])]
2778    ""
2779 {
2780   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2781   operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2782   operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2783 })
2784
2785 (define_insn "*add<mode>3_carryinV_zero"
2786   [(set (reg:CC_V CC_REGNUM)
2787         (compare:CC_V
2788           (plus:<DWI>
2789             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2790             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2791           (sign_extend:<DWI>
2792             (plus:GPI
2793               (match_operand:GPI 3 "aarch64_carry_operation" "")
2794               (match_dup 1)))))
2795    (set (match_operand:GPI 0 "register_operand" "=r")
2796         (plus:GPI (match_dup 3) (match_dup 1)))]
2797    ""
2798    "adcs\\t%<w>0, %<w>1, <w>zr"
2799   [(set_attr "type" "adc_reg")]
2800 )
2801
2802 (define_insn "*add<mode>3_carryinV"
2803   [(set (reg:CC_V CC_REGNUM)
2804         (compare:CC_V
2805           (plus:<DWI>
2806             (plus:<DWI>
2807               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2808               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2809             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2810           (sign_extend:<DWI>
2811             (plus:GPI
2812               (plus:GPI
2813                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2814                 (match_dup 1))
2815               (match_dup 2)))))
2816    (set (match_operand:GPI 0 "register_operand" "=r")
2817         (plus:GPI
2818           (plus:GPI (match_dup 4) (match_dup 1))
2819           (match_dup 2)))]
2820    ""
2821    "adcs\\t%<w>0, %<w>1, %<w>2"
2822   [(set_attr "type" "adc_reg")]
2823 )
2824
2825 (define_insn "*add_uxt<mode>_shift2"
2826   [(set (match_operand:GPI 0 "register_operand" "=rk")
2827         (plus:GPI (and:GPI
2828                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2829                                (match_operand 2 "aarch64_imm3" "Ui3"))
2830                    (match_operand 3 "const_int_operand" "n"))
2831                   (match_operand:GPI 4 "register_operand" "r")))]
2832   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2833   "*
2834   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2835                                            INTVAL (operands[3])));
2836   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2837   [(set_attr "type" "alu_ext")]
2838 )
2839
2840 ;; zero_extend version of above
2841 (define_insn "*add_uxtsi_shift2_uxtw"
2842   [(set (match_operand:DI 0 "register_operand" "=rk")
2843         (zero_extend:DI
2844          (plus:SI (and:SI
2845                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2846                               (match_operand 2 "aarch64_imm3" "Ui3"))
2847                    (match_operand 3 "const_int_operand" "n"))
2848                   (match_operand:SI 4 "register_operand" "r"))))]
2849   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2850   "*
2851   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2852                                            INTVAL (operands[3])));
2853   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2854   [(set_attr "type" "alu_ext")]
2855 )
2856
2857 (define_insn "*add_uxt<mode>_multp2"
2858   [(set (match_operand:GPI 0 "register_operand" "=rk")
2859         (plus:GPI (and:GPI
2860                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2861                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2862                    (match_operand 3 "const_int_operand" "n"))
2863                   (match_operand:GPI 4 "register_operand" "r")))]
2864   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2865   "*
2866   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2867                                            INTVAL (operands[3])));
2868   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2869   [(set_attr "type" "alu_ext")]
2870 )
2871
2872 ;; zero_extend version of above
2873 (define_insn "*add_uxtsi_multp2_uxtw"
2874   [(set (match_operand:DI 0 "register_operand" "=rk")
2875         (zero_extend:DI
2876          (plus:SI (and:SI
2877                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2878                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2879                    (match_operand 3 "const_int_operand" "n"))
2880                   (match_operand:SI 4 "register_operand" "r"))))]
2881   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2882   "*
2883   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2884                                            INTVAL (operands[3])));
2885   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2886   [(set_attr "type" "alu_ext")]
2887 )
2888
2889 (define_insn "subsi3"
2890   [(set (match_operand:SI 0 "register_operand" "=rk")
2891         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2892                   (match_operand:SI 2 "register_operand" "r")))]
2893   ""
2894   "sub\\t%w0, %w1, %w2"
2895   [(set_attr "type" "alu_sreg")]
2896 )
2897
2898 ;; zero_extend version of above
2899 (define_insn "*subsi3_uxtw"
2900   [(set (match_operand:DI 0 "register_operand" "=rk")
2901         (zero_extend:DI
2902          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2903                    (match_operand:SI 2 "register_operand" "r"))))]
2904   ""
2905   "sub\\t%w0, %w1, %w2"
2906   [(set_attr "type" "alu_sreg")]
2907 )
2908
2909 (define_insn "subdi3"
2910   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2911         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2912                   (match_operand:DI 2 "register_operand" "r,w")))]
2913   ""
2914   "@
2915    sub\\t%x0, %x1, %x2
2916    sub\\t%d0, %d1, %d2"
2917   [(set_attr "type" "alu_sreg, neon_sub")
2918    (set_attr "arch" "*,simd")]
2919 )
2920
2921 (define_expand "subv<GPI:mode>4"
2922   [(match_operand:GPI 0 "register_operand")
2923    (match_operand:GPI 1 "register_operand")
2924    (match_operand:GPI 2 "aarch64_plus_operand")
2925    (label_ref (match_operand 3 "" ""))]
2926   ""
2927 {
2928   if (CONST_INT_P (operands[2]))
2929     emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2930   else
2931     emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2932   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2933
2934   DONE;
2935 })
2936
2937 (define_insn "subv<GPI:mode>_insn"
2938   [(set (reg:CC_V CC_REGNUM)
2939         (compare:CC_V
2940          (sign_extend:<DWI>
2941           (minus:GPI
2942            (match_operand:GPI 1 "register_operand" "rk")
2943            (match_operand:GPI 2 "register_operand" "r")))
2944          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2945                       (sign_extend:<DWI> (match_dup 2)))))
2946    (set (match_operand:GPI 0 "register_operand" "=r")
2947         (minus:GPI (match_dup 1) (match_dup 2)))]
2948   ""
2949   "subs\\t%<w>0, %<w>1, %<w>2"
2950   [(set_attr "type" "alus_sreg")]
2951 )
2952
2953 (define_insn "subv<GPI:mode>_imm"
2954   [(set (reg:CC_V CC_REGNUM)
2955         (compare:CC_V
2956          (sign_extend:<DWI>
2957           (minus:GPI
2958            (match_operand:GPI 1 "register_operand" "rk,rk")
2959            (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2960          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2961                       (match_dup 2))))
2962    (set (match_operand:GPI 0 "register_operand" "=r,r")
2963         (minus:GPI (match_dup 1) (match_dup 2)))]
2964   ""
2965   "@
2966    subs\\t%<w>0, %<w>1, %2
2967    adds\\t%<w>0, %<w>1, #%n2"
2968   [(set_attr "type" "alus_sreg")]
2969 )
2970
2971 (define_expand "negv<GPI:mode>3"
2972   [(match_operand:GPI 0 "register_operand")
2973    (match_operand:GPI 1 "register_operand")
2974    (label_ref (match_operand 2 "" ""))]
2975   ""
2976   {
2977     emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2978     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2979
2980     DONE;
2981   }
2982 )
2983
2984 (define_insn "negv<GPI:mode>_insn"
2985   [(set (reg:CC_V CC_REGNUM)
2986         (compare:CC_V
2987          (sign_extend:<DWI>
2988           (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2989          (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2990    (set (match_operand:GPI 0 "register_operand" "=r")
2991         (neg:GPI (match_dup 1)))]
2992   ""
2993   "negs\\t%<w>0, %<w>1"
2994   [(set_attr "type" "alus_sreg")]
2995 )
2996
2997 (define_insn "negv<GPI:mode>_cmp_only"
2998   [(set (reg:CC_V CC_REGNUM)
2999         (compare:CC_V
3000          (sign_extend:<DWI>
3001           (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
3002          (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
3003   ""
3004   "negs\\t%<w>zr, %<w>0"
3005   [(set_attr "type" "alus_sreg")]
3006 )
3007
3008 (define_insn "*cmpv<GPI:mode>_insn"
3009   [(set (reg:CC_V CC_REGNUM)
3010         (compare:CC_V
3011          (sign_extend:<DWI>
3012           (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
3013                      (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
3014          (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
3015                     (sign_extend:<DWI> (match_dup 1)))))]
3016   ""
3017   "@
3018    cmp\\t%<w>0, %<w>1
3019    cmp\\t%<w>0, %1
3020    cmp\\t%<w>0, #%n1"
3021   [(set_attr "type" "alus_sreg")]
3022 )
3023
3024 (define_expand "usubv<mode>4"
3025   [(match_operand:GPI 0 "register_operand")
3026    (match_operand:GPI 1 "aarch64_reg_or_zero")
3027    (match_operand:GPI 2 "aarch64_reg_or_zero")
3028    (label_ref (match_operand 3 "" ""))]
3029   ""
3030 {
3031   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
3032   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
3033
3034   DONE;
3035 })
3036
3037 (define_expand "subti3"
3038   [(set (match_operand:TI 0 "register_operand")
3039         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
3040                   (match_operand:TI 2 "register_operand")))]
3041   ""
3042 {
3043   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3044
3045   aarch64_subvti_scratch_regs (operands[1], operands[2],
3046                                &low_dest, &op1_low, &op2_low,
3047                                &high_dest, &op1_high, &op2_high);
3048
3049   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
3050   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
3051
3052   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
3053   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
3054   DONE;
3055 })
3056
3057 (define_expand "subvti4"
3058   [(match_operand:TI 0 "register_operand")
3059    (match_operand:TI 1 "register_operand")
3060    (match_operand:TI 2 "aarch64_reg_or_imm")
3061    (label_ref (match_operand 3 "" ""))]
3062   ""
3063 {
3064   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3065
3066   aarch64_subvti_scratch_regs (operands[1], operands[2],
3067                                &low_dest, &op1_low, &op2_low,
3068                                &high_dest, &op1_high, &op2_high);
3069   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
3070                          high_dest, op1_high, op2_high, false);
3071
3072   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
3073   DONE;
3074 })
3075
3076 (define_expand "usubvti4"
3077   [(match_operand:TI 0 "register_operand")
3078    (match_operand:TI 1 "register_operand")
3079    (match_operand:TI 2 "aarch64_reg_or_imm")
3080    (label_ref (match_operand 3 "" ""))]
3081   ""
3082 {
3083   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3084
3085   aarch64_subvti_scratch_regs (operands[1], operands[2],
3086                                     &low_dest, &op1_low, &op2_low,
3087                                &high_dest, &op1_high, &op2_high);
3088   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
3089                          high_dest, op1_high, op2_high, true);
3090
3091   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
3092   DONE;
3093 })
3094
3095 (define_expand "negvti3"
3096   [(match_operand:TI 0 "register_operand")
3097    (match_operand:TI 1 "register_operand")
3098    (label_ref (match_operand 2 "" ""))]
3099   ""
3100   {
3101     emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
3102                                    gen_lowpart (DImode, operands[1])));
3103     emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
3104                                     gen_highpart (DImode, operands[1])));
3105     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
3106
3107     DONE;
3108   }
3109 )
3110
3111 (define_insn "negdi_carryout"
3112   [(set (reg:CC CC_REGNUM)
3113         (compare:CC
3114          (const_int 0) (match_operand:DI 1 "register_operand" "r")))
3115    (set (match_operand:DI 0 "register_operand" "=r")
3116         (neg:DI (match_dup 1)))]
3117   ""
3118   "negs\\t%0, %1"
3119   [(set_attr "type" "alus_sreg")]
3120 )
3121
3122 (define_insn "negvdi_carryinV"
3123   [(set (reg:CC_V CC_REGNUM)
3124         (compare:CC_V
3125          (neg:TI (plus:TI
3126                   (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
3127                   (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
3128          (sign_extend:TI
3129           (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3130                            (match_dup 1))))))
3131    (set (match_operand:DI 0 "register_operand" "=r")
3132         (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3133                          (match_dup 1))))]
3134   ""
3135   "ngcs\\t%0, %1"
3136   [(set_attr "type" "alus_sreg")]
3137 )
3138
3139 (define_insn "*sub<mode>3_compare0"
3140   [(set (reg:CC_NZ CC_REGNUM)
3141         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3142                                   (match_operand:GPI 2 "register_operand" "r"))
3143                        (const_int 0)))
3144    (set (match_operand:GPI 0 "register_operand" "=r")
3145         (minus:GPI (match_dup 1) (match_dup 2)))]
3146   ""
3147   "subs\\t%<w>0, %<w>1, %<w>2"
3148   [(set_attr "type" "alus_sreg")]
3149 )
3150
3151 ;; zero_extend version of above
3152 (define_insn "*subsi3_compare0_uxtw"
3153   [(set (reg:CC_NZ CC_REGNUM)
3154         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3155                                  (match_operand:SI 2 "register_operand" "r"))
3156                        (const_int 0)))
3157    (set (match_operand:DI 0 "register_operand" "=r")
3158         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3159   ""
3160   "subs\\t%w0, %w1, %w2"
3161   [(set_attr "type" "alus_sreg")]
3162 )
3163
3164 (define_insn "sub<mode>3_compare1_imm"
3165   [(set (reg:CC CC_REGNUM)
3166         (compare:CC
3167           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
3168           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3169    (set (match_operand:GPI 0 "register_operand" "=r,r")
3170         (plus:GPI
3171           (match_dup 1)
3172           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3173   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3174   "@
3175   subs\\t%<w>0, %<w>1, %2
3176   adds\\t%<w>0, %<w>1, #%n2"
3177   [(set_attr "type" "alus_imm")]
3178 )
3179
3180 (define_insn "sub<mode>3_compare1"
3181   [(set (reg:CC CC_REGNUM)
3182         (compare:CC
3183           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3184           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3185    (set (match_operand:GPI 0 "register_operand" "=r")
3186         (minus:GPI (match_dup 1) (match_dup 2)))]
3187   ""
3188   "subs\\t%<w>0, %<w>1, %<w>2"
3189   [(set_attr "type" "alus_sreg")]
3190 )
3191
3192 (define_peephole2
3193   [(set (match_operand:GPI 0 "aarch64_general_reg")
3194         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3195                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
3196    (set (reg:CC CC_REGNUM)
3197         (compare:CC
3198           (match_dup 1)
3199           (match_dup 2)))]
3200   "!reg_overlap_mentioned_p (operands[0], operands[1])
3201    && !reg_overlap_mentioned_p (operands[0], operands[2])"
3202   [(const_int 0)]
3203   {
3204     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3205                                          operands[2]));
3206     DONE;
3207   }
3208 )
3209
3210 ;; Same as the above peephole but with the compare and minus in
3211 ;; swapped order.  The restriction on overlap between operand 0
3212 ;; and operands 1 and 2 doesn't apply here.
3213 (define_peephole2
3214   [(set (reg:CC CC_REGNUM)
3215         (compare:CC
3216           (match_operand:GPI 1 "aarch64_reg_or_zero")
3217           (match_operand:GPI 2 "aarch64_reg_or_zero")))
3218    (set (match_operand:GPI 0 "aarch64_general_reg")
3219         (minus:GPI (match_dup 1)
3220                    (match_dup 2)))]
3221   ""
3222   [(const_int 0)]
3223   {
3224     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3225                                          operands[2]));
3226     DONE;
3227   }
3228 )
3229
3230 (define_peephole2
3231   [(set (match_operand:GPI 0 "aarch64_general_reg")
3232         (plus:GPI (match_operand:GPI 1 "register_operand")
3233                   (match_operand:GPI 2 "aarch64_plus_immediate")))
3234    (set (reg:CC CC_REGNUM)
3235         (compare:CC
3236           (match_dup 1)
3237           (match_operand:GPI 3 "const_int_operand")))]
3238   "!reg_overlap_mentioned_p (operands[0], operands[1])
3239    && INTVAL (operands[3]) == -INTVAL (operands[2])"
3240   [(const_int 0)]
3241   {
3242     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3243                                          operands[3], operands[2]));
3244     DONE;
3245   }
3246 )
3247
3248 ;; Same as the above peephole but with the compare and minus in
3249 ;; swapped order.  The restriction on overlap between operand 0
3250 ;; and operands 1 doesn't apply here.
3251 (define_peephole2
3252   [(set (reg:CC CC_REGNUM)
3253         (compare:CC
3254           (match_operand:GPI 1 "register_operand")
3255           (match_operand:GPI 3 "const_int_operand")))
3256    (set (match_operand:GPI 0 "aarch64_general_reg")
3257         (plus:GPI (match_dup 1)
3258                   (match_operand:GPI 2 "aarch64_plus_immediate")))]
3259   "INTVAL (operands[3]) == -INTVAL (operands[2])"
3260   [(const_int 0)]
3261   {
3262     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3263                                          operands[3], operands[2]));
3264     DONE;
3265   }
3266 )
3267
3268 (define_insn "*sub_<shift>_<mode>"
3269   [(set (match_operand:GPI 0 "register_operand" "=r")
3270         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3271                    (ASHIFT:GPI
3272                     (match_operand:GPI 1 "register_operand" "r")
3273                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3274   ""
3275   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3276   [(set_attr "type" "alu_shift_imm")]
3277 )
3278
3279 ;; zero_extend version of above
3280 (define_insn "*sub_<shift>_si_uxtw"
3281   [(set (match_operand:DI 0 "register_operand" "=r")
3282         (zero_extend:DI
3283          (minus:SI (match_operand:SI 3 "register_operand" "r")
3284                    (ASHIFT:SI
3285                     (match_operand:SI 1 "register_operand" "r")
3286                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3287   ""
3288   "sub\\t%w0, %w3, %w1, <shift> %2"
3289   [(set_attr "type" "alu_shift_imm")]
3290 )
3291
3292 (define_insn "*sub_mul_imm_<mode>"
3293   [(set (match_operand:GPI 0 "register_operand" "=r")
3294         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3295                    (mult:GPI
3296                     (match_operand:GPI 1 "register_operand" "r")
3297                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3298   ""
3299   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3300   [(set_attr "type" "alu_shift_imm")]
3301 )
3302
3303 ;; zero_extend version of above
3304 (define_insn "*sub_mul_imm_si_uxtw"
3305   [(set (match_operand:DI 0 "register_operand" "=r")
3306         (zero_extend:DI
3307          (minus:SI (match_operand:SI 3 "register_operand" "r")
3308                    (mult:SI
3309                     (match_operand:SI 1 "register_operand" "r")
3310                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3311   ""
3312   "sub\\t%w0, %w3, %w1, lsl %p2"
3313   [(set_attr "type" "alu_shift_imm")]
3314 )
3315
3316 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3317   [(set (match_operand:GPI 0 "register_operand" "=rk")
3318         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3319                    (ANY_EXTEND:GPI
3320                     (match_operand:ALLX 2 "register_operand" "r"))))]
3321   ""
3322   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3323   [(set_attr "type" "alu_ext")]
3324 )
3325
3326 ;; zero_extend version of above
3327 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3328   [(set (match_operand:DI 0 "register_operand" "=rk")
3329         (zero_extend:DI
3330          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3331                    (ANY_EXTEND:SI
3332                     (match_operand:SHORT 2 "register_operand" "r")))))]
3333   ""
3334   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3335   [(set_attr "type" "alu_ext")]
3336 )
3337
3338 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3339   [(set (match_operand:GPI 0 "register_operand" "=rk")
3340         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3341                    (ashift:GPI (ANY_EXTEND:GPI
3342                                 (match_operand:ALLX 2 "register_operand" "r"))
3343                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
3344   ""
3345   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3346   [(set_attr "type" "alu_ext")]
3347 )
3348
3349 ;; zero_extend version of above
3350 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3351   [(set (match_operand:DI 0 "register_operand" "=rk")
3352         (zero_extend:DI
3353          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3354                    (ashift:SI (ANY_EXTEND:SI
3355                                (match_operand:SHORT 2 "register_operand" "r"))
3356                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
3357   ""
3358   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3359   [(set_attr "type" "alu_ext")]
3360 )
3361
3362 (define_insn "*sub_<optab><mode>_multp2"
3363   [(set (match_operand:GPI 0 "register_operand" "=rk")
3364         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3365                    (ANY_EXTRACT:GPI
3366                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3367                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3368                     (match_operand 3 "const_int_operand" "n")
3369                     (const_int 0))))]
3370   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3371   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3372   [(set_attr "type" "alu_ext")]
3373 )
3374
3375 ;; zero_extend version of above
3376 (define_insn "*sub_<optab>si_multp2_uxtw"
3377   [(set (match_operand:DI 0 "register_operand" "=rk")
3378         (zero_extend:DI
3379          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3380                    (ANY_EXTRACT:SI
3381                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3382                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3383                     (match_operand 3 "const_int_operand" "n")
3384                     (const_int 0)))))]
3385   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3386   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3387   [(set_attr "type" "alu_ext")]
3388 )
3389
3390 ;; The hardware description is op1 + ~op2 + C.
3391 ;;                           = op1 + (-op2 + 1) + (1 - !C)
3392 ;;                           = op1 - op2 - 1 + 1 - !C
3393 ;;                           = op1 - op2 - !C.
3394 ;; We describe the latter.
3395
3396 (define_insn "*sub<mode>3_carryin0"
3397   [(set (match_operand:GPI 0 "register_operand" "=r")
3398         (minus:GPI
3399           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3400           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3401    ""
3402    "sbc\\t%<w>0, %<w>1, <w>zr"
3403   [(set_attr "type" "adc_reg")]
3404 )
3405
3406 ;; zero_extend version of the above
3407 (define_insn "*subsi3_carryin_uxtw"
3408   [(set (match_operand:DI 0 "register_operand" "=r")
3409         (zero_extend:DI
3410           (minus:SI
3411             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3412             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3413    ""
3414    "sbc\\t%w0, %w1, wzr"
3415   [(set_attr "type" "adc_reg")]
3416 )
3417
3418 (define_expand "sub<mode>3_carryin"
3419   [(set (match_operand:GPI 0 "register_operand")
3420         (minus:GPI
3421           (minus:GPI
3422             (match_operand:GPI 1 "aarch64_reg_or_zero")
3423             (match_operand:GPI 2 "register_operand"))
3424           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3425    ""
3426    ""
3427 )
3428
3429 (define_insn "*sub<mode>3_carryin"
3430   [(set (match_operand:GPI 0 "register_operand" "=r")
3431         (minus:GPI
3432           (minus:GPI
3433             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3434             (match_operand:GPI 2 "register_operand" "r"))
3435           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3436
3437    ""
3438    "sbc\\t%<w>0, %<w>1, %<w>2"
3439   [(set_attr "type" "adc_reg")]
3440 )
3441
3442 ;; zero_extend version of the above
3443 (define_insn "*subsi3_carryin_uxtw"
3444   [(set (match_operand:DI 0 "register_operand" "=r")
3445         (zero_extend:DI
3446           (minus:SI
3447             (minus:SI
3448               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3449               (match_operand:SI 2 "register_operand" "r"))
3450             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3451
3452    ""
3453    "sbc\\t%w0, %w1, %w2"
3454   [(set_attr "type" "adc_reg")]
3455 )
3456
3457 (define_insn "*sub<mode>3_carryin_alt"
3458   [(set (match_operand:GPI 0 "register_operand" "=r")
3459         (minus:GPI
3460           (minus:GPI
3461             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3462             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3463           (match_operand:GPI 2 "register_operand" "r")))]
3464    ""
3465    "sbc\\t%<w>0, %<w>1, %<w>2"
3466   [(set_attr "type" "adc_reg")]
3467 )
3468
3469 ;; zero_extend version of the above
3470 (define_insn "*subsi3_carryin_alt_uxtw"
3471   [(set (match_operand:DI 0 "register_operand" "=r")
3472         (zero_extend:DI
3473           (minus:SI
3474             (minus:SI
3475               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3476               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3477             (match_operand:SI 2 "register_operand" "r"))))]
3478    ""
3479    "sbc\\t%w0, %w1, %w2"
3480   [(set_attr "type" "adc_reg")]
3481 )
3482
3483 (define_expand "usub<GPI:mode>3_carryinC"
3484   [(parallel
3485      [(set (reg:CC CC_REGNUM)
3486            (compare:CC
3487              (zero_extend:<DWI>
3488                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3489              (plus:<DWI>
3490                (zero_extend:<DWI>
3491                  (match_operand:GPI 2 "register_operand"))
3492                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3493       (set (match_operand:GPI 0 "register_operand")
3494            (minus:GPI
3495              (minus:GPI (match_dup 1) (match_dup 2))
3496              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3497    ""
3498 )
3499
3500 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3501   [(set (reg:CC CC_REGNUM)
3502         (compare:CC
3503           (const_int 0)
3504           (plus:<DWI>
3505             (zero_extend:<DWI>
3506               (match_operand:GPI 1 "register_operand" "r"))
3507             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3508    (set (match_operand:GPI 0 "register_operand" "=r")
3509         (minus:GPI
3510           (neg:GPI (match_dup 1))
3511           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3512    ""
3513    "sbcs\\t%<w>0, <w>zr, %<w>1"
3514   [(set_attr "type" "adc_reg")]
3515 )
3516
3517 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3518   [(set (reg:CC CC_REGNUM)
3519         (compare:CC
3520           (zero_extend:<DWI>
3521             (match_operand:GPI 1 "register_operand" "r"))
3522           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3523    (set (match_operand:GPI 0 "register_operand" "=r")
3524         (minus:GPI
3525           (match_dup 1)
3526           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3527    ""
3528    "sbcs\\t%<w>0, %<w>1, <w>zr"
3529   [(set_attr "type" "adc_reg")]
3530 )
3531
3532 (define_insn "*usub<GPI:mode>3_carryinC"
3533   [(set (reg:CC CC_REGNUM)
3534         (compare:CC
3535           (zero_extend:<DWI>
3536             (match_operand:GPI 1 "register_operand" "r"))
3537           (plus:<DWI>
3538             (zero_extend:<DWI>
3539               (match_operand:GPI 2 "register_operand" "r"))
3540             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3541    (set (match_operand:GPI 0 "register_operand" "=r")
3542         (minus:GPI
3543           (minus:GPI (match_dup 1) (match_dup 2))
3544           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3545    ""
3546    "sbcs\\t%<w>0, %<w>1, %<w>2"
3547   [(set_attr "type" "adc_reg")]
3548 )
3549
3550 (define_expand "sub<GPI:mode>3_carryinV"
3551   [(parallel
3552      [(set (reg:CC_V CC_REGNUM)
3553            (compare:CC_V
3554             (minus:<DWI>
3555              (sign_extend:<DWI>
3556                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3557              (plus:<DWI>
3558                (sign_extend:<DWI>
3559                  (match_operand:GPI 2 "register_operand"))
3560                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3561             (sign_extend:<DWI>
3562              (minus:GPI (match_dup 1)
3563                         (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3564                                   (match_dup 2))))))
3565       (set (match_operand:GPI 0 "register_operand")
3566            (minus:GPI
3567              (minus:GPI (match_dup 1) (match_dup 2))
3568              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3569    ""
3570 )
3571
3572 (define_insn "*sub<mode>3_carryinV_z2"
3573   [(set (reg:CC_V CC_REGNUM)
3574         (compare:CC_V
3575          (minus:<DWI>
3576           (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3577           (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3578          (sign_extend:<DWI>
3579           (minus:GPI (match_dup 1)
3580                      (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3581    (set (match_operand:GPI 0 "register_operand" "=r")
3582         (minus:GPI
3583          (match_dup 1) (match_dup 3)))]
3584    ""
3585    "sbcs\\t%<w>0, %<w>1, <w>zr"
3586   [(set_attr "type" "adc_reg")]
3587 )
3588
3589 (define_insn "*sub<mode>3_carryinV"
3590   [(set (reg:CC_V CC_REGNUM)
3591         (compare:CC_V
3592          (minus:<DWI>
3593           (sign_extend:<DWI>
3594             (match_operand:GPI 1 "register_operand" "r"))
3595           (plus:<DWI>
3596             (sign_extend:<DWI>
3597               (match_operand:GPI 2 "register_operand" "r"))
3598             (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3599          (sign_extend:<DWI>
3600           (minus:GPI
3601            (match_dup 1)
3602            (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3603                      (match_dup 2))))))
3604    (set (match_operand:GPI 0 "register_operand" "=r")
3605         (minus:GPI
3606           (minus:GPI (match_dup 1) (match_dup 2))
3607           (match_dup 4)))]
3608    ""
3609    "sbcs\\t%<w>0, %<w>1, %<w>2"
3610   [(set_attr "type" "adc_reg")]
3611 )
3612
3613 (define_insn "*sub_uxt<mode>_shift2"
3614   [(set (match_operand:GPI 0 "register_operand" "=rk")
3615         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3616                    (and:GPI
3617                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3618                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3619                     (match_operand 3 "const_int_operand" "n"))))]
3620   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3621   "*
3622   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3623                                            INTVAL (operands[3])));
3624   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3625   [(set_attr "type" "alu_ext")]
3626 )
3627
3628 ;; zero_extend version of above
3629 (define_insn "*sub_uxtsi_shift2_uxtw"
3630   [(set (match_operand:DI 0 "register_operand" "=rk")
3631         (zero_extend:DI
3632          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3633                    (and:SI
3634                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3635                                (match_operand 2 "aarch64_imm3" "Ui3"))
3636                     (match_operand 3 "const_int_operand" "n")))))]
3637   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3638   "*
3639   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3640                                            INTVAL (operands[3])));
3641   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3642   [(set_attr "type" "alu_ext")]
3643 )
3644
3645 (define_insn "*sub_uxt<mode>_multp2"
3646   [(set (match_operand:GPI 0 "register_operand" "=rk")
3647         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3648                    (and:GPI
3649                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3650                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3651                     (match_operand 3 "const_int_operand" "n"))))]
3652   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3653   "*
3654   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3655                                            INTVAL (operands[3])));
3656   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3657   [(set_attr "type" "alu_ext")]
3658 )
3659
3660 ;; zero_extend version of above
3661 (define_insn "*sub_uxtsi_multp2_uxtw"
3662   [(set (match_operand:DI 0 "register_operand" "=rk")
3663         (zero_extend:DI
3664          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3665                    (and:SI
3666                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3667                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3668                     (match_operand 3 "const_int_operand" "n")))))]
3669   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3670   "*
3671   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3672                                            INTVAL (operands[3])));
3673   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3674   [(set_attr "type" "alu_ext")]
3675 )
3676
3677 (define_expand "abs<mode>2"
3678   [(match_operand:GPI 0 "register_operand")
3679    (match_operand:GPI 1 "register_operand")]
3680   ""
3681   {
3682     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3683     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3684     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3685     DONE;
3686   }
3687 )
3688
3689 (define_insn "neg<mode>2"
3690   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3691         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3692   ""
3693   "@
3694    neg\\t%<w>0, %<w>1
3695    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3696   [(set_attr "type" "alu_sreg, neon_neg<q>")
3697    (set_attr "arch" "*,simd")]
3698 )
3699
3700 ;; zero_extend version of above
3701 (define_insn "*negsi2_uxtw"
3702   [(set (match_operand:DI 0 "register_operand" "=r")
3703         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3704   ""
3705   "neg\\t%w0, %w1"
3706   [(set_attr "type" "alu_sreg")]
3707 )
3708
3709 (define_insn "*ngc<mode>"
3710   [(set (match_operand:GPI 0 "register_operand" "=r")
3711         (minus:GPI
3712           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3713           (match_operand:GPI 1 "register_operand" "r")))]
3714   ""
3715   "ngc\\t%<w>0, %<w>1"
3716   [(set_attr "type" "adc_reg")]
3717 )
3718
3719 (define_insn "*ngcsi_uxtw"
3720   [(set (match_operand:DI 0 "register_operand" "=r")
3721         (zero_extend:DI
3722           (minus:SI
3723             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3724             (match_operand:SI 1 "register_operand" "r"))))]
3725   ""
3726   "ngc\\t%w0, %w1"
3727   [(set_attr "type" "adc_reg")]
3728 )
3729
3730 (define_insn "neg<mode>2_compare0"
3731   [(set (reg:CC_NZ CC_REGNUM)
3732         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3733                        (const_int 0)))
3734    (set (match_operand:GPI 0 "register_operand" "=r")
3735         (neg:GPI (match_dup 1)))]
3736   ""
3737   "negs\\t%<w>0, %<w>1"
3738   [(set_attr "type" "alus_sreg")]
3739 )
3740
3741 ;; zero_extend version of above
3742 (define_insn "*negsi2_compare0_uxtw"
3743   [(set (reg:CC_NZ CC_REGNUM)
3744         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3745                        (const_int 0)))
3746    (set (match_operand:DI 0 "register_operand" "=r")
3747         (zero_extend:DI (neg:SI (match_dup 1))))]
3748   ""
3749   "negs\\t%w0, %w1"
3750   [(set_attr "type" "alus_sreg")]
3751 )
3752
3753 (define_insn "*neg_<shift><mode>3_compare0"
3754   [(set (reg:CC_NZ CC_REGNUM)
3755         (compare:CC_NZ
3756          (neg:GPI (ASHIFT:GPI
3757                    (match_operand:GPI 1 "register_operand" "r")
3758                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3759          (const_int 0)))
3760    (set (match_operand:GPI 0 "register_operand" "=r")
3761         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3762   ""
3763   "negs\\t%<w>0, %<w>1, <shift> %2"
3764   [(set_attr "type" "alus_shift_imm")]
3765 )
3766
3767 (define_insn "*neg_<shift>_<mode>2"
3768   [(set (match_operand:GPI 0 "register_operand" "=r")
3769         (neg:GPI (ASHIFT:GPI
3770                   (match_operand:GPI 1 "register_operand" "r")
3771                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3772   ""
3773   "neg\\t%<w>0, %<w>1, <shift> %2"
3774   [(set_attr "type" "alu_shift_imm")]
3775 )
3776
3777 ;; zero_extend version of above
3778 (define_insn "*neg_<shift>_si2_uxtw"
3779   [(set (match_operand:DI 0 "register_operand" "=r")
3780         (zero_extend:DI
3781          (neg:SI (ASHIFT:SI
3782                   (match_operand:SI 1 "register_operand" "r")
3783                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3784   ""
3785   "neg\\t%w0, %w1, <shift> %2"
3786   [(set_attr "type" "alu_shift_imm")]
3787 )
3788
3789 (define_insn "*neg_mul_imm_<mode>2"
3790   [(set (match_operand:GPI 0 "register_operand" "=r")
3791         (neg:GPI (mult:GPI
3792                   (match_operand:GPI 1 "register_operand" "r")
3793                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3794   ""
3795   "neg\\t%<w>0, %<w>1, lsl %p2"
3796   [(set_attr "type" "alu_shift_imm")]
3797 )
3798
3799 ;; zero_extend version of above
3800 (define_insn "*neg_mul_imm_si2_uxtw"
3801   [(set (match_operand:DI 0 "register_operand" "=r")
3802         (zero_extend:DI
3803          (neg:SI (mult:SI
3804                   (match_operand:SI 1 "register_operand" "r")
3805                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3806   ""
3807   "neg\\t%w0, %w1, lsl %p2"
3808   [(set_attr "type" "alu_shift_imm")]
3809 )
3810
3811 (define_insn "mul<mode>3"
3812   [(set (match_operand:GPI 0 "register_operand" "=r")
3813         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3814                   (match_operand:GPI 2 "register_operand" "r")))]
3815   ""
3816   "mul\\t%<w>0, %<w>1, %<w>2"
3817   [(set_attr "type" "mul")]
3818 )
3819
3820 ;; zero_extend version of above
3821 (define_insn "*mulsi3_uxtw"
3822   [(set (match_operand:DI 0 "register_operand" "=r")
3823         (zero_extend:DI
3824          (mult:SI (match_operand:SI 1 "register_operand" "r")
3825                   (match_operand:SI 2 "register_operand" "r"))))]
3826   ""
3827   "mul\\t%w0, %w1, %w2"
3828   [(set_attr "type" "mul")]
3829 )
3830
3831 (define_insn "madd<mode>"
3832   [(set (match_operand:GPI 0 "register_operand" "=r")
3833         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3834                             (match_operand:GPI 2 "register_operand" "r"))
3835                   (match_operand:GPI 3 "register_operand" "r")))]
3836   ""
3837   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3838   [(set_attr "type" "mla")]
3839 )
3840
3841 ;; zero_extend version of above
3842 (define_insn "*maddsi_uxtw"
3843   [(set (match_operand:DI 0 "register_operand" "=r")
3844         (zero_extend:DI
3845          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3846                            (match_operand:SI 2 "register_operand" "r"))
3847                   (match_operand:SI 3 "register_operand" "r"))))]
3848   ""
3849   "madd\\t%w0, %w1, %w2, %w3"
3850   [(set_attr "type" "mla")]
3851 )
3852
3853 (define_insn "*msub<mode>"
3854   [(set (match_operand:GPI 0 "register_operand" "=r")
3855         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3856                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3857                              (match_operand:GPI 2 "register_operand" "r"))))]
3858
3859   ""
3860   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3861   [(set_attr "type" "mla")]
3862 )
3863
3864 ;; zero_extend version of above
3865 (define_insn "*msubsi_uxtw"
3866   [(set (match_operand:DI 0 "register_operand" "=r")
3867         (zero_extend:DI
3868          (minus:SI (match_operand:SI 3 "register_operand" "r")
3869                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3870                             (match_operand:SI 2 "register_operand" "r")))))]
3871
3872   ""
3873   "msub\\t%w0, %w1, %w2, %w3"
3874   [(set_attr "type" "mla")]
3875 )
3876
3877 (define_insn "*mul<mode>_neg"
3878   [(set (match_operand:GPI 0 "register_operand" "=r")
3879         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3880                   (match_operand:GPI 2 "register_operand" "r")))]
3881
3882   ""
3883   "mneg\\t%<w>0, %<w>1, %<w>2"
3884   [(set_attr "type" "mul")]
3885 )
3886
3887 ;; zero_extend version of above
3888 (define_insn "*mulsi_neg_uxtw"
3889   [(set (match_operand:DI 0 "register_operand" "=r")
3890         (zero_extend:DI
3891          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3892                   (match_operand:SI 2 "register_operand" "r"))))]
3893
3894   ""
3895   "mneg\\t%w0, %w1, %w2"
3896   [(set_attr "type" "mul")]
3897 )
3898
3899 (define_insn "<su_optab>mulsidi3"
3900   [(set (match_operand:DI 0 "register_operand" "=r")
3901         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3902                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3903   ""
3904   "<su>mull\\t%0, %w1, %w2"
3905   [(set_attr "type" "<su>mull")]
3906 )
3907
3908 (define_insn "<su_optab>maddsidi4"
3909   [(set (match_operand:DI 0 "register_operand" "=r")
3910         (plus:DI (mult:DI
3911                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3912                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3913                  (match_operand:DI 3 "register_operand" "r")))]
3914   ""
3915   "<su>maddl\\t%0, %w1, %w2, %3"
3916   [(set_attr "type" "<su>mlal")]
3917 )
3918
3919 (define_insn "<su_optab>msubsidi4"
3920   [(set (match_operand:DI 0 "register_operand" "=r")
3921         (minus:DI
3922          (match_operand:DI 3 "register_operand" "r")
3923          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3924                   (ANY_EXTEND:DI
3925                    (match_operand:SI 2 "register_operand" "r")))))]
3926   ""
3927   "<su>msubl\\t%0, %w1, %w2, %3"
3928   [(set_attr "type" "<su>mlal")]
3929 )
3930
3931 (define_insn "*<su_optab>mulsidi_neg"
3932   [(set (match_operand:DI 0 "register_operand" "=r")
3933         (mult:DI (neg:DI
3934                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3935                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3936   ""
3937   "<su>mnegl\\t%0, %w1, %w2"
3938   [(set_attr "type" "<su>mull")]
3939 )
3940
3941 (define_expand "<su_optab>mulditi3"
3942   [(set (match_operand:TI 0 "register_operand")
3943         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3944                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3945   ""
3946 {
3947   rtx low = gen_reg_rtx (DImode);
3948   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3949
3950   rtx high = gen_reg_rtx (DImode);
3951   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3952
3953   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3954   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3955   DONE;
3956 })
3957
3958 ;; The default expansion of multi3 using umuldi3_highpart will perform
3959 ;; the additions in an order that fails to combine into two madd insns.
3960 (define_expand "multi3"
3961   [(set (match_operand:TI 0 "register_operand")
3962         (mult:TI (match_operand:TI 1 "register_operand")
3963                  (match_operand:TI 2 "register_operand")))]
3964   ""
3965 {
3966   rtx l0 = gen_reg_rtx (DImode);
3967   rtx l1 = gen_lowpart (DImode, operands[1]);
3968   rtx l2 = gen_lowpart (DImode, operands[2]);
3969   rtx h0 = gen_reg_rtx (DImode);
3970   rtx h1 = gen_highpart (DImode, operands[1]);
3971   rtx h2 = gen_highpart (DImode, operands[2]);
3972
3973   emit_insn (gen_muldi3 (l0, l1, l2));
3974   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3975   emit_insn (gen_madddi (h0, h1, l2, h0));
3976   emit_insn (gen_madddi (h0, l1, h2, h0));
3977
3978   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3979   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3980   DONE;
3981 })
3982
3983 (define_insn "<su>muldi3_highpart"
3984   [(set (match_operand:DI 0 "register_operand" "=r")
3985         (truncate:DI
3986          (lshiftrt:TI
3987           (mult:TI
3988            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3989            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3990           (const_int 64))))]
3991   ""
3992   "<su>mulh\\t%0, %1, %2"
3993   [(set_attr "type" "<su>mull")]
3994 )
3995
3996 (define_insn "<su_optab>div<mode>3"
3997   [(set (match_operand:GPI 0 "register_operand" "=r")
3998         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3999                      (match_operand:GPI 2 "register_operand" "r")))]
4000   ""
4001   "<su>div\\t%<w>0, %<w>1, %<w>2"
4002   [(set_attr "type" "<su>div")]
4003 )
4004
4005 ;; zero_extend version of above
4006 (define_insn "*<su_optab>divsi3_uxtw"
4007   [(set (match_operand:DI 0 "register_operand" "=r")
4008         (zero_extend:DI
4009          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
4010                      (match_operand:SI 2 "register_operand" "r"))))]
4011   ""
4012   "<su>div\\t%w0, %w1, %w2"
4013   [(set_attr "type" "<su>div")]
4014 )
4015
4016 ;; -------------------------------------------------------------------
4017 ;; Comparison insns
4018 ;; -------------------------------------------------------------------
4019
4020 (define_insn "cmp<mode>"
4021   [(set (reg:CC CC_REGNUM)
4022         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
4023                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
4024   ""
4025   "@
4026    cmp\\t%<w>0, %<w>1
4027    cmp\\t%<w>0, %1
4028    cmn\\t%<w>0, #%n1"
4029   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
4030 )
4031
4032 (define_insn "fcmp<mode>"
4033   [(set (reg:CCFP CC_REGNUM)
4034         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
4035                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
4036    "TARGET_FLOAT"
4037    "@
4038     fcmp\\t%<s>0, #0.0
4039     fcmp\\t%<s>0, %<s>1"
4040   [(set_attr "type" "fcmp<s>")]
4041 )
4042
4043 (define_insn "fcmpe<mode>"
4044   [(set (reg:CCFPE CC_REGNUM)
4045         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
4046                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
4047    "TARGET_FLOAT"
4048    "@
4049     fcmpe\\t%<s>0, #0.0
4050     fcmpe\\t%<s>0, %<s>1"
4051   [(set_attr "type" "fcmp<s>")]
4052 )
4053
4054 (define_insn "*cmp_swp_<shift>_reg<mode>"
4055   [(set (reg:CC_SWP CC_REGNUM)
4056         (compare:CC_SWP (ASHIFT:GPI
4057                          (match_operand:GPI 0 "register_operand" "r")
4058                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4059                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
4060   ""
4061   "cmp\\t%<w>2, %<w>0, <shift> %1"
4062   [(set_attr "type" "alus_shift_imm")]
4063 )
4064
4065 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
4066   [(set (reg:CC_SWP CC_REGNUM)
4067         (compare:CC_SWP (ANY_EXTEND:GPI
4068                          (match_operand:ALLX 0 "register_operand" "r"))
4069                         (match_operand:GPI 1 "register_operand" "r")))]
4070   ""
4071   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
4072   [(set_attr "type" "alus_ext")]
4073 )
4074
4075 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
4076   [(set (reg:CC_SWP CC_REGNUM)
4077         (compare:CC_SWP (ashift:GPI
4078                          (ANY_EXTEND:GPI
4079                           (match_operand:ALLX 0 "register_operand" "r"))
4080                          (match_operand 1 "aarch64_imm3" "Ui3"))
4081         (match_operand:GPI 2 "register_operand" "r")))]
4082   ""
4083   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
4084   [(set_attr "type" "alus_ext")]
4085 )
4086
4087 ;; -------------------------------------------------------------------
4088 ;; Store-flag and conditional select insns
4089 ;; -------------------------------------------------------------------
4090
4091 (define_expand "cstore<mode>4"
4092   [(set (match_operand:SI 0 "register_operand")
4093         (match_operator:SI 1 "aarch64_comparison_operator"
4094          [(match_operand:GPI 2 "register_operand")
4095           (match_operand:GPI 3 "aarch64_plus_operand")]))]
4096   ""
4097   "
4098   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4099                                       operands[3]);
4100   operands[3] = const0_rtx;
4101   "
4102 )
4103
4104 (define_expand "cstorecc4"
4105   [(set (match_operand:SI 0 "register_operand")
4106        (match_operator 1 "aarch64_comparison_operator_mode"
4107         [(match_operand 2 "cc_register")
4108          (match_operand 3 "const0_operand")]))]
4109   ""
4110 "{
4111   emit_insn (gen_rtx_SET (operands[0], operands[1]));
4112   DONE;
4113 }")
4114
4115
4116 (define_expand "cstore<mode>4"
4117   [(set (match_operand:SI 0 "register_operand")
4118         (match_operator:SI 1 "aarch64_comparison_operator_mode"
4119          [(match_operand:GPF 2 "register_operand")
4120           (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
4121   ""
4122   "
4123   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4124                                       operands[3]);
4125   operands[3] = const0_rtx;
4126   "
4127 )
4128
4129 (define_insn "aarch64_cstore<mode>"
4130   [(set (match_operand:ALLI 0 "register_operand" "=r")
4131         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4132          [(match_operand 2 "cc_register" "") (const_int 0)]))]
4133   ""
4134   "cset\\t%<w>0, %m1"
4135   [(set_attr "type" "csel")]
4136 )
4137
4138 ;; For a 24-bit immediate CST we can optimize the compare for equality
4139 ;; and branch sequence from:
4140 ;;      mov     x0, #imm1
4141 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
4142 ;;      cmp     x1, x0
4143 ;;      cset    x2, <ne,eq>
4144 ;; into the shorter:
4145 ;;      sub     x0, x1, #(CST & 0xfff000)
4146 ;;      subs    x0, x0, #(CST & 0x000fff)
4147 ;;      cset x2, <ne, eq>.
4148 (define_insn_and_split "*compare_cstore<mode>_insn"
4149   [(set (match_operand:GPI 0 "register_operand" "=r")
4150          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
4151                   (match_operand:GPI 2 "aarch64_imm24" "n")))
4152    (clobber (reg:CC CC_REGNUM))]
4153   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4154    && !aarch64_plus_operand (operands[2], <MODE>mode)
4155    && !reload_completed"
4156   "#"
4157   "&& true"
4158   [(const_int 0)]
4159   {
4160     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4161     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4162     rtx tmp = gen_reg_rtx (<MODE>mode);
4163     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4164     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4165     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4166     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4167     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4168     DONE;
4169   }
4170   [(set_attr "type" "csel")]
4171 )
4172
4173 ;; zero_extend version of the above
4174 (define_insn "*cstoresi_insn_uxtw"
4175   [(set (match_operand:DI 0 "register_operand" "=r")
4176         (zero_extend:DI
4177          (match_operator:SI 1 "aarch64_comparison_operator_mode"
4178           [(match_operand 2 "cc_register" "") (const_int 0)])))]
4179   ""
4180   "cset\\t%w0, %m1"
4181   [(set_attr "type" "csel")]
4182 )
4183
4184 (define_insn "cstore<mode>_neg"
4185   [(set (match_operand:ALLI 0 "register_operand" "=r")
4186         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4187                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
4188   ""
4189   "csetm\\t%<w>0, %m1"
4190   [(set_attr "type" "csel")]
4191 )
4192
4193 ;; zero_extend version of the above
4194 (define_insn "*cstoresi_neg_uxtw"
4195   [(set (match_operand:DI 0 "register_operand" "=r")
4196         (zero_extend:DI
4197          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4198                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4199   ""
4200   "csetm\\t%w0, %m1"
4201   [(set_attr "type" "csel")]
4202 )
4203
4204 (define_expand "cmov<mode>6"
4205   [(set (match_operand:GPI 0 "register_operand")
4206         (if_then_else:GPI
4207          (match_operator 1 "aarch64_comparison_operator"
4208           [(match_operand:GPI 2 "register_operand")
4209            (match_operand:GPI 3 "aarch64_plus_operand")])
4210          (match_operand:GPI 4 "register_operand")
4211          (match_operand:GPI 5 "register_operand")))]
4212   ""
4213   "
4214   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4215                                       operands[3]);
4216   operands[3] = const0_rtx;
4217   "
4218 )
4219
4220 (define_expand "cmov<mode>6"
4221   [(set (match_operand:GPF 0 "register_operand")
4222         (if_then_else:GPF
4223          (match_operator 1 "aarch64_comparison_operator"
4224           [(match_operand:GPF 2 "register_operand")
4225            (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4226          (match_operand:GPF 4 "register_operand")
4227          (match_operand:GPF 5 "register_operand")))]
4228   ""
4229   "
4230   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4231                                       operands[3]);
4232   operands[3] = const0_rtx;
4233   "
4234 )
4235
4236 (define_insn "*cmov<mode>_insn"
4237   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4238         (if_then_else:ALLI
4239          (match_operator 1 "aarch64_comparison_operator"
4240           [(match_operand 2 "cc_register" "") (const_int 0)])
4241          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4242          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4243   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4244      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4245   ;; Final two alternatives should be unreachable, but included for completeness
4246   "@
4247    csel\\t%<w>0, %<w>3, %<w>4, %m1
4248    csinv\\t%<w>0, %<w>3, <w>zr, %m1
4249    csinv\\t%<w>0, %<w>4, <w>zr, %M1
4250    csinc\\t%<w>0, %<w>3, <w>zr, %m1
4251    csinc\\t%<w>0, %<w>4, <w>zr, %M1
4252    mov\\t%<w>0, -1
4253    mov\\t%<w>0, 1"
4254   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4255 )
4256
4257 ;; zero_extend version of above
4258 (define_insn "*cmovsi_insn_uxtw"
4259   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4260         (zero_extend:DI
4261          (if_then_else:SI
4262           (match_operator 1 "aarch64_comparison_operator"
4263            [(match_operand 2 "cc_register" "") (const_int 0)])
4264           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4265           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4266   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4267      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4268   ;; Final two alternatives should be unreachable, but included for completeness
4269   "@
4270    csel\\t%w0, %w3, %w4, %m1
4271    csinv\\t%w0, %w3, wzr, %m1
4272    csinv\\t%w0, %w4, wzr, %M1
4273    csinc\\t%w0, %w3, wzr, %m1
4274    csinc\\t%w0, %w4, wzr, %M1
4275    mov\\t%w0, -1
4276    mov\\t%w0, 1"
4277   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4278 )
4279
4280 (define_insn "*cmovdi_insn_uxtw"
4281   [(set (match_operand:DI 0 "register_operand" "=r")
4282         (if_then_else:DI
4283          (match_operator 1 "aarch64_comparison_operator"
4284           [(match_operand 2 "cc_register" "") (const_int 0)])
4285          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4286          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4287   ""
4288   "csel\\t%w0, %w3, %w4, %m1"
4289   [(set_attr "type" "csel")]
4290 )
4291
4292 (define_insn "*cmov<mode>_insn"
4293   [(set (match_operand:GPF 0 "register_operand" "=w")
4294         (if_then_else:GPF
4295          (match_operator 1 "aarch64_comparison_operator"
4296           [(match_operand 2 "cc_register" "") (const_int 0)])
4297          (match_operand:GPF 3 "register_operand" "w")
4298          (match_operand:GPF 4 "register_operand" "w")))]
4299   "TARGET_FLOAT"
4300   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4301   [(set_attr "type" "fcsel")]
4302 )
4303
4304 (define_expand "mov<mode>cc"
4305   [(set (match_operand:ALLI 0 "register_operand")
4306         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4307                            (match_operand:ALLI 2 "register_operand")
4308                            (match_operand:ALLI 3 "register_operand")))]
4309   ""
4310   {
4311     rtx ccreg;
4312     enum rtx_code code = GET_CODE (operands[1]);
4313
4314     if (code == UNEQ || code == LTGT)
4315       FAIL;
4316
4317     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4318                                      XEXP (operands[1], 1));
4319     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4320   }
4321 )
4322
4323 (define_expand "mov<GPF:mode><GPI:mode>cc"
4324   [(set (match_operand:GPI 0 "register_operand")
4325         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4326                           (match_operand:GPF 2 "register_operand")
4327                           (match_operand:GPF 3 "register_operand")))]
4328   ""
4329   {
4330     rtx ccreg;
4331     enum rtx_code code = GET_CODE (operands[1]);
4332
4333     if (code == UNEQ || code == LTGT)
4334       FAIL;
4335
4336     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4337                                   XEXP (operands[1], 1));
4338     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4339   }
4340 )
4341
4342 (define_expand "mov<mode>cc"
4343   [(set (match_operand:GPF 0 "register_operand")
4344         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4345                           (match_operand:GPF 2 "register_operand")
4346                           (match_operand:GPF 3 "register_operand")))]
4347   ""
4348   {
4349     rtx ccreg;
4350     enum rtx_code code = GET_CODE (operands[1]);
4351
4352     if (code == UNEQ || code == LTGT)
4353       FAIL;
4354
4355     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4356                                   XEXP (operands[1], 1));
4357     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4358   }
4359 )
4360
4361 (define_expand "<neg_not_op><mode>cc"
4362   [(set (match_operand:GPI 0 "register_operand")
4363         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4364                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4365                           (match_operand:GPI 3 "register_operand")))]
4366   ""
4367   {
4368     rtx ccreg;
4369     enum rtx_code code = GET_CODE (operands[1]);
4370
4371     if (code == UNEQ || code == LTGT)
4372       FAIL;
4373
4374     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4375                                       XEXP (operands[1], 1));
4376     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4377   }
4378 )
4379
4380 ;; CRC32 instructions.
4381 (define_insn "aarch64_<crc_variant>"
4382   [(set (match_operand:SI 0 "register_operand" "=r")
4383         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4384                     (match_operand:<crc_mode> 2 "register_operand" "r")]
4385          CRC))]
4386   "TARGET_CRC32"
4387   {
4388     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4389       return "<crc_variant>\\t%w0, %w1, %x2";
4390     else
4391       return "<crc_variant>\\t%w0, %w1, %w2";
4392   }
4393   [(set_attr "type" "crc")]
4394 )
4395
4396 (define_insn "*csinc2<mode>_insn"
4397   [(set (match_operand:GPI 0 "register_operand" "=r")
4398         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4399                   (match_operand:GPI 1 "register_operand" "r")))]
4400   ""
4401   "cinc\\t%<w>0, %<w>1, %m2"
4402   [(set_attr "type" "csel")]
4403 )
4404
4405 (define_insn "csinc3<mode>_insn"
4406   [(set (match_operand:GPI 0 "register_operand" "=r")
4407         (if_then_else:GPI
4408           (match_operand 1 "aarch64_comparison_operation" "")
4409           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4410                     (const_int 1))
4411           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4412   ""
4413   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4414   [(set_attr "type" "csel")]
4415 )
4416
4417 (define_insn "*csinv3<mode>_insn"
4418   [(set (match_operand:GPI 0 "register_operand" "=r")
4419         (if_then_else:GPI
4420           (match_operand 1 "aarch64_comparison_operation" "")
4421           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4422           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4423   ""
4424   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4425   [(set_attr "type" "csel")]
4426 )
4427
4428 (define_insn "csneg3_uxtw_insn"
4429   [(set (match_operand:DI 0 "register_operand" "=r")
4430         (zero_extend:DI
4431           (if_then_else:SI
4432             (match_operand 1 "aarch64_comparison_operation" "")
4433             (neg:SI (match_operand:SI 2 "register_operand" "r"))
4434             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4435   ""
4436   "csneg\\t%w0, %w3, %w2, %M1"
4437   [(set_attr "type" "csel")]
4438 )
4439
4440 (define_insn "csneg3<mode>_insn"
4441   [(set (match_operand:GPI 0 "register_operand" "=r")
4442         (if_then_else:GPI
4443           (match_operand 1 "aarch64_comparison_operation" "")
4444           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4445           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4446   ""
4447   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4448   [(set_attr "type" "csel")]
4449 )
4450
4451 (define_insn "*csinv3_uxtw_insn1"
4452   [(set (match_operand:DI 0 "register_operand" "=r")
4453         (if_then_else:DI
4454           (match_operand 1 "aarch64_comparison_operation" "")
4455           (zero_extend:DI
4456             (match_operand:SI 2 "register_operand" "r"))
4457           (zero_extend:DI
4458             (NEG_NOT:SI (match_operand:SI 3 "register_operand" "r")))))]
4459   ""
4460   "cs<neg_not_cs>\\t%w0, %w2, %w3, %m1"
4461   [(set_attr "type" "csel")]
4462 )
4463
4464 (define_insn "*csinv3_uxtw_insn2"
4465   [(set (match_operand:DI 0 "register_operand" "=r")
4466         (if_then_else:DI
4467           (match_operand 1 "aarch64_comparison_operation" "")
4468           (zero_extend:DI
4469             (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4470           (zero_extend:DI
4471             (match_operand:SI 3 "register_operand" "r"))))]
4472   ""
4473   "cs<neg_not_cs>\\t%w0, %w3, %w2, %M1"
4474   [(set_attr "type" "csel")]
4475 )
4476
4477 (define_insn "*csinv3_uxtw_insn3"
4478   [(set (match_operand:DI 0 "register_operand" "=r")
4479         (if_then_else:DI
4480           (match_operand 1 "aarch64_comparison_operation" "")
4481           (zero_extend:DI
4482             (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4483           (const_int 0)))]
4484   ""
4485   "cs<neg_not_cs>\\t%w0, wzr, %w2, %M1"
4486   [(set_attr "type" "csel")]
4487 )
4488
4489 ;; If X can be loaded by a single CNT[BHWD] instruction,
4490 ;;
4491 ;;    A = UMAX (B, X)
4492 ;;
4493 ;; is equivalent to:
4494 ;;
4495 ;;    TMP = UQDEC[BHWD] (B, X)
4496 ;;    A = TMP + X
4497 ;;
4498 ;; Defining the pattern this way means that:
4499 ;;
4500 ;;    A = UMAX (B, X) - X
4501 ;;
4502 ;; becomes:
4503 ;;
4504 ;;    TMP1 = UQDEC[BHWD] (B, X)
4505 ;;    TMP2 = TMP1 + X
4506 ;;    A = TMP2 - X
4507 ;;
4508 ;; which combine can optimize to:
4509 ;;
4510 ;;    A = UQDEC[BHWD] (B, X)
4511 ;;
4512 ;; We don't use match_operand predicates because the order of the operands
4513 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4514 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4515 (define_expand "umax<mode>3"
4516   [(set (match_operand:GPI 0 "register_operand")
4517         (umax:GPI (match_operand:GPI 1 "")
4518                   (match_operand:GPI 2 "")))]
4519   "TARGET_SVE"
4520   {
4521     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4522       std::swap (operands[1], operands[2]);
4523     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4524       FAIL;
4525     rtx temp = gen_reg_rtx (<MODE>mode);
4526     operands[1] = force_reg (<MODE>mode, operands[1]);
4527     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4528     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4529     DONE;
4530   }
4531 )
4532
4533 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4534 (define_insn "aarch64_uqdec<mode>"
4535   [(set (match_operand:GPI 0 "register_operand" "=r")
4536         (minus:GPI
4537          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4538                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4539          (match_dup 2)))]
4540   "TARGET_SVE"
4541   {
4542     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4543   }
4544 )
4545
4546 ;; -------------------------------------------------------------------
4547 ;; Logical operations
4548 ;; -------------------------------------------------------------------
4549
4550
4551 (define_insn_and_split "*aarch64_and<mode>_imm2"
4552   [(set (match_operand:GPI 0 "register_operand" "=rk")
4553         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4554                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4555   ""
4556   "#"
4557   "true"
4558   [(const_int 0)]
4559   {
4560      HOST_WIDE_INT val = INTVAL (operands[2]);
4561      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4562      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4563
4564      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4565      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4566      DONE;
4567   }
4568 )
4569
4570 (define_insn "<optab><mode>3"
4571   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4572         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4573                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4574   ""
4575   "@
4576   <logical>\\t%<w>0, %<w>1, %<w>2
4577   <logical>\\t%<w>0, %<w>1, %2
4578   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4579   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4580    (set_attr "arch" "*,*,simd")]
4581 )
4582
4583 ;; zero_extend version of above
4584 (define_insn "*<optab>si3_uxtw"
4585   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4586         (zero_extend:DI
4587          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4588                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4589   ""
4590   "@
4591    <logical>\\t%w0, %w1, %w2
4592    <logical>\\t%w0, %w1, %2"
4593   [(set_attr "type" "logic_reg,logic_imm")]
4594 )
4595
4596 (define_insn "*and<mode>3_compare0"
4597   [(set (reg:CC_NZ CC_REGNUM)
4598         (compare:CC_NZ
4599          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4600                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4601          (const_int 0)))
4602    (set (match_operand:GPI 0 "register_operand" "=r,r")
4603         (and:GPI (match_dup 1) (match_dup 2)))]
4604   ""
4605   "@
4606    ands\\t%<w>0, %<w>1, %<w>2
4607    ands\\t%<w>0, %<w>1, %2"
4608   [(set_attr "type" "logics_reg,logics_imm")]
4609 )
4610
4611 ;; zero_extend version of above
4612 (define_insn "*andsi3_compare0_uxtw"
4613   [(set (reg:CC_NZ CC_REGNUM)
4614         (compare:CC_NZ
4615          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4616                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4617          (const_int 0)))
4618    (set (match_operand:DI 0 "register_operand" "=r,r")
4619         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4620   ""
4621   "@
4622    ands\\t%w0, %w1, %w2
4623    ands\\t%w0, %w1, %2"
4624   [(set_attr "type" "logics_reg,logics_imm")]
4625 )
4626
4627 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4628   [(set (reg:CC_NZ CC_REGNUM)
4629         (compare:CC_NZ
4630          (and:GPI (SHIFT:GPI
4631                    (match_operand:GPI 1 "register_operand" "r")
4632                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4633                   (match_operand:GPI 3 "register_operand" "r"))
4634          (const_int 0)))
4635    (set (match_operand:GPI 0 "register_operand" "=r")
4636         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4637   ""
4638   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4639   [(set_attr "type" "logics_shift_imm")]
4640 )
4641
4642 ;; zero_extend version of above
4643 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4644   [(set (reg:CC_NZ CC_REGNUM)
4645         (compare:CC_NZ
4646          (and:SI (SHIFT:SI
4647                   (match_operand:SI 1 "register_operand" "r")
4648                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4649                  (match_operand:SI 3 "register_operand" "r"))
4650          (const_int 0)))
4651    (set (match_operand:DI 0 "register_operand" "=r")
4652         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4653                                 (match_dup 3))))]
4654   ""
4655   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4656   [(set_attr "type" "logics_shift_imm")]
4657 )
4658
4659 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4660   [(set (match_operand:GPI 0 "register_operand" "=r")
4661         (LOGICAL:GPI (SHIFT:GPI
4662                       (match_operand:GPI 1 "register_operand" "r")
4663                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4664                      (match_operand:GPI 3 "register_operand" "r")))]
4665   ""
4666   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4667   [(set_attr "type" "logic_shift_imm")]
4668 )
4669
4670 (define_insn "*<optab>_rol<mode>3"
4671   [(set (match_operand:GPI 0 "register_operand" "=r")
4672         (LOGICAL:GPI (rotate:GPI
4673                       (match_operand:GPI 1 "register_operand" "r")
4674                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4675                      (match_operand:GPI 3 "register_operand" "r")))]
4676   ""
4677   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4678   [(set_attr "type" "logic_shift_imm")]
4679 )
4680
4681 ;; zero_extend versions of above
4682 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4683   [(set (match_operand:DI 0 "register_operand" "=r")
4684         (zero_extend:DI
4685          (LOGICAL:SI (SHIFT:SI
4686                       (match_operand:SI 1 "register_operand" "r")
4687                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4688                      (match_operand:SI 3 "register_operand" "r"))))]
4689   ""
4690   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4691   [(set_attr "type" "logic_shift_imm")]
4692 )
4693
4694 (define_insn "*<optab>_rolsi3_uxtw"
4695   [(set (match_operand:DI 0 "register_operand" "=r")
4696         (zero_extend:DI
4697          (LOGICAL:SI (rotate:SI
4698                       (match_operand:SI 1 "register_operand" "r")
4699                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4700                      (match_operand:SI 3 "register_operand" "r"))))]
4701   ""
4702   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4703   [(set_attr "type" "logic_shift_imm")]
4704 )
4705
4706 (define_insn "one_cmpl<mode>2"
4707   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4708         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4709   ""
4710   "@
4711   mvn\\t%<w>0, %<w>1
4712   mvn\\t%0.8b, %1.8b"
4713   [(set_attr "type" "logic_reg,neon_logic")
4714    (set_attr "arch" "*,simd")]
4715 )
4716
4717 (define_insn "*one_cmpl_zero_extend"
4718   [(set (match_operand:DI 0 "register_operand" "=r")
4719         (zero_extend:DI
4720           (not:SI (match_operand:SI 1 "register_operand" "r"))))]
4721   ""
4722   "mvn\\t%w0, %w1"
4723   [(set_attr "type" "logic_reg")]
4724 )
4725
4726 (define_insn "*one_cmpl_<optab><mode>2"
4727   [(set (match_operand:GPI 0 "register_operand" "=r")
4728         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4729                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4730   ""
4731   "mvn\\t%<w>0, %<w>1, <shift> %2"
4732   [(set_attr "type" "logic_shift_imm")]
4733 )
4734
4735 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4736
4737 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4738   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4739         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4740                      (match_operand:GPI 2 "register_operand" "r,w")))]
4741   ""
4742   "@
4743   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4744   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4745   [(set_attr "type" "logic_reg,neon_logic")
4746    (set_attr "arch" "*,simd")]
4747 )
4748
4749 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4750   [(set (match_operand:DI 0 "register_operand" "=r")
4751         (zero_extend:DI
4752           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4753                        (match_operand:SI 2 "register_operand" "r"))))]
4754   ""
4755   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4756   [(set_attr "type" "logic_reg")]
4757 )
4758
4759 (define_insn "*xor_one_cmplsidi3_ze"
4760   [(set (match_operand:DI 0 "register_operand" "=r")
4761         (zero_extend:DI
4762           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4763                           (match_operand:SI 2 "register_operand" "r")))))]
4764   ""
4765   "eon\\t%w0, %w1, %w2"
4766   [(set_attr "type" "logic_reg")]
4767 )
4768
4769 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4770 ;; eon does not operate on SIMD registers so the vector variant must be split.
4771 (define_insn_and_split "*xor_one_cmpl<mode>3"
4772   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4773         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4774                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4775   ""
4776   "@
4777   eon\\t%<w>0, %<w>1, %<w>2
4778   #"
4779   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4780   [(set (match_operand:GPI 0 "register_operand" "=w")
4781         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4782                  (match_operand:GPI 2 "register_operand" "w")))
4783    (set (match_dup 0) (not:GPI (match_dup 0)))]
4784   ""
4785   [(set_attr "type" "logic_reg,multiple")
4786    (set_attr "arch" "*,simd")]
4787 )
4788
4789 (define_insn "*and_one_cmpl<mode>3_compare0"
4790   [(set (reg:CC_NZ CC_REGNUM)
4791         (compare:CC_NZ
4792          (and:GPI (not:GPI
4793                    (match_operand:GPI 1 "register_operand" "r"))
4794                   (match_operand:GPI 2 "register_operand" "r"))
4795          (const_int 0)))
4796    (set (match_operand:GPI 0 "register_operand" "=r")
4797         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4798   ""
4799   "bics\\t%<w>0, %<w>2, %<w>1"
4800   [(set_attr "type" "logics_reg")]
4801 )
4802
4803 ;; zero_extend version of above
4804 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4805   [(set (reg:CC_NZ CC_REGNUM)
4806         (compare:CC_NZ
4807          (and:SI (not:SI
4808                   (match_operand:SI 1 "register_operand" "r"))
4809                  (match_operand:SI 2 "register_operand" "r"))
4810          (const_int 0)))
4811    (set (match_operand:DI 0 "register_operand" "=r")
4812         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4813   ""
4814   "bics\\t%w0, %w2, %w1"
4815   [(set_attr "type" "logics_reg")]
4816 )
4817
4818 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4819   [(set (reg:CC_NZ CC_REGNUM)
4820     (compare:CC_NZ
4821      (and:GPI (not:GPI
4822            (match_operand:GPI 0 "register_operand" "r"))
4823           (match_operand:GPI 1 "register_operand" "r"))
4824      (const_int 0)))]
4825   ""
4826   "bics\\t<w>zr, %<w>1, %<w>0"
4827   [(set_attr "type" "logics_reg")]
4828 )
4829
4830 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4831   [(set (match_operand:GPI 0 "register_operand" "=r")
4832         (LOGICAL:GPI (not:GPI
4833                       (SHIFT:GPI
4834                        (match_operand:GPI 1 "register_operand" "r")
4835                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4836                      (match_operand:GPI 3 "register_operand" "r")))]
4837   ""
4838   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4839   [(set_attr "type" "logic_shift_imm")]
4840 )
4841
4842 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4843   [(set (match_operand:GPI 0 "register_operand" "=r")
4844         (not:GPI (xor:GPI
4845                       (SHIFT:GPI
4846                        (match_operand:GPI 1 "register_operand" "r")
4847                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4848                      (match_operand:GPI 3 "register_operand" "r"))))]
4849   ""
4850   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4851   [(set_attr "type" "logic_shift_imm")]
4852 )
4853
4854 ;; Zero-extend version of the above.
4855 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4856   [(set (match_operand:DI 0 "register_operand" "=r")
4857         (zero_extend:DI
4858           (not:SI (xor:SI
4859                     (SHIFT:SI
4860                       (match_operand:SI 1 "register_operand" "r")
4861                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4862                     (match_operand:SI 3 "register_operand" "r")))))]
4863   ""
4864   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4865   [(set_attr "type" "logic_shift_imm")]
4866 )
4867
4868 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4869   [(set (reg:CC_NZ CC_REGNUM)
4870         (compare:CC_NZ
4871          (and:GPI (not:GPI
4872                    (SHIFT:GPI
4873                     (match_operand:GPI 1 "register_operand" "r")
4874                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4875                   (match_operand:GPI 3 "register_operand" "r"))
4876          (const_int 0)))
4877    (set (match_operand:GPI 0 "register_operand" "=r")
4878         (and:GPI (not:GPI
4879                   (SHIFT:GPI
4880                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4881   ""
4882   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4883   [(set_attr "type" "logics_shift_imm")]
4884 )
4885
4886 ;; zero_extend version of above
4887 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4888   [(set (reg:CC_NZ CC_REGNUM)
4889         (compare:CC_NZ
4890          (and:SI (not:SI
4891                   (SHIFT:SI
4892                    (match_operand:SI 1 "register_operand" "r")
4893                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4894                  (match_operand:SI 3 "register_operand" "r"))
4895          (const_int 0)))
4896    (set (match_operand:DI 0 "register_operand" "=r")
4897         (zero_extend:DI (and:SI
4898                          (not:SI
4899                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4900   ""
4901   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4902   [(set_attr "type" "logics_shift_imm")]
4903 )
4904
4905 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4906   [(set (reg:CC_NZ CC_REGNUM)
4907     (compare:CC_NZ
4908      (and:GPI (not:GPI
4909            (SHIFT:GPI
4910             (match_operand:GPI 0 "register_operand" "r")
4911             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4912           (match_operand:GPI 2 "register_operand" "r"))
4913      (const_int 0)))]
4914   ""
4915   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4916   [(set_attr "type" "logics_shift_imm")]
4917 )
4918
4919 (define_insn "clz<mode>2"
4920   [(set (match_operand:GPI 0 "register_operand" "=r")
4921         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4922   ""
4923   "clz\\t%<w>0, %<w>1"
4924   [(set_attr "type" "clz")]
4925 )
4926
4927 (define_expand "ffs<mode>2"
4928   [(match_operand:GPI 0 "register_operand")
4929    (match_operand:GPI 1 "register_operand")]
4930   ""
4931   {
4932     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4933     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4934
4935     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4936     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4937     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4938     DONE;
4939   }
4940 )
4941
4942 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4943 ;;
4944 ;; MOV  v.1d, x0
4945 ;; CNT  v1.8b, v.8b
4946 ;; ADDV b2, v1.8b
4947 ;; MOV  w0, v2.b[0]
4948
4949 (define_expand "popcount<mode>2"
4950   [(match_operand:GPI 0 "register_operand")
4951    (match_operand:GPI 1 "register_operand")]
4952   "TARGET_SIMD"
4953 {
4954   rtx v = gen_reg_rtx (V8QImode);
4955   rtx v1 = gen_reg_rtx (V8QImode);
4956   rtx in = operands[1];
4957   rtx out = operands[0];
4958   if(<MODE>mode == SImode)
4959     {
4960       rtx tmp;
4961       tmp = gen_reg_rtx (DImode);
4962       /* If we have SImode, zero extend to DImode, pop count does
4963          not change if we have extra zeros. */
4964       emit_insn (gen_zero_extendsidi2 (tmp, in));
4965       in = tmp;
4966     }
4967   emit_move_insn (v, gen_lowpart (V8QImode, in));
4968   emit_insn (gen_popcountv8qi2 (v1, v));
4969   emit_insn (gen_aarch64_zero_extend<mode>_reduc_plus_v8qi (out, v1));
4970   DONE;
4971 })
4972
4973 (define_insn "clrsb<mode>2"
4974   [(set (match_operand:GPI 0 "register_operand" "=r")
4975         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4976   ""
4977   "cls\\t%<w>0, %<w>1"
4978   [(set_attr "type" "clz")]
4979 )
4980
4981 (define_insn "rbit<mode>2"
4982   [(set (match_operand:GPI 0 "register_operand" "=r")
4983         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4984   ""
4985   "rbit\\t%<w>0, %<w>1"
4986   [(set_attr "type" "rbit")]
4987 )
4988
4989 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4990 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4991 ;; expression and split after reload to enable scheduling them apart if
4992 ;; needed.
4993
4994 (define_insn_and_split "ctz<mode>2"
4995  [(set (match_operand:GPI           0 "register_operand" "=r")
4996        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4997   ""
4998   "#"
4999   "reload_completed"
5000   [(const_int 0)]
5001   "
5002   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
5003   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
5004   DONE;
5005 ")
5006
5007 (define_insn "*and<mode>_compare0"
5008   [(set (reg:CC_NZ CC_REGNUM)
5009         (compare:CC_NZ
5010          (match_operand:SHORT 0 "register_operand" "r")
5011          (const_int 0)))]
5012   ""
5013   "tst\\t%<w>0, <short_mask>"
5014   [(set_attr "type" "alus_imm")]
5015 )
5016
5017 (define_insn "*ands<GPI:mode>_compare0"
5018   [(set (reg:CC_NZ CC_REGNUM)
5019         (compare:CC_NZ
5020          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
5021          (const_int 0)))
5022    (set (match_operand:GPI 0 "register_operand" "=r")
5023         (zero_extend:GPI (match_dup 1)))]
5024   ""
5025   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
5026   [(set_attr "type" "alus_imm")]
5027 )
5028
5029 (define_insn "*and<mode>3nr_compare0"
5030   [(set (reg:CC_NZ CC_REGNUM)
5031         (compare:CC_NZ
5032          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
5033                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
5034          (const_int 0)))]
5035   ""
5036   "@
5037    tst\\t%<w>0, %<w>1
5038    tst\\t%<w>0, %1"
5039   [(set_attr "type" "logics_reg,logics_imm")]
5040 )
5041
5042 (define_split
5043   [(set (reg:CC_NZ CC_REGNUM)
5044         (compare:CC_NZ
5045          (and:GPI (match_operand:GPI 0 "register_operand")
5046                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
5047          (const_int 0)))
5048    (clobber (match_operand:SI 2 "register_operand"))]
5049   ""
5050   [(set (match_dup 2) (match_dup 1))
5051    (set (reg:CC_NZ CC_REGNUM)
5052         (compare:CC_NZ
5053          (and:GPI (match_dup 0)
5054                   (match_dup 2))
5055          (const_int 0)))]
5056 )
5057
5058 (define_insn "*and<mode>3nr_compare0_zextract"
5059   [(set (reg:CC_NZ CC_REGNUM)
5060         (compare:CC_NZ
5061          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
5062                   (match_operand:GPI 1 "const_int_operand" "n")
5063                   (match_operand:GPI 2 "const_int_operand" "n"))
5064          (const_int 0)))]
5065   "INTVAL (operands[1]) > 0
5066    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
5067         <= GET_MODE_BITSIZE (<MODE>mode))
5068    && aarch64_bitmask_imm (
5069         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
5070                                                  operands[2])),
5071         <MODE>mode)"
5072   {
5073     operands[1]
5074       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
5075     return "tst\\t%<w>0, %1";
5076   }
5077   [(set_attr "type" "logics_shift_imm")]
5078 )
5079
5080 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
5081   [(set (reg:CC_NZ CC_REGNUM)
5082         (compare:CC_NZ
5083          (and:GPI (SHIFT:GPI
5084                    (match_operand:GPI 0 "register_operand" "r")
5085                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
5086                   (match_operand:GPI 2 "register_operand" "r"))
5087         (const_int 0)))]
5088   ""
5089   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
5090   [(set_attr "type" "logics_shift_imm")]
5091 )
5092
5093 (define_split
5094   [(set (reg:CC_NZ CC_REGNUM)
5095         (compare:CC_NZ
5096          (and:GPI (SHIFT:GPI
5097                    (match_operand:GPI 0 "register_operand")
5098                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
5099                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
5100         (const_int 0)))
5101     (clobber (match_operand:SI 3 "register_operand"))]
5102   ""
5103   [(set (match_dup 3) (match_dup 2))
5104    (set (reg:CC_NZ CC_REGNUM)
5105         (compare:CC_NZ
5106          (and:GPI (SHIFT:GPI
5107                    (match_dup 0)
5108                    (match_dup 1))
5109                   (match_dup 3))
5110          (const_int 0)))]
5111 )
5112
5113 ;; -------------------------------------------------------------------
5114 ;; Shifts
5115 ;; -------------------------------------------------------------------
5116
5117 (define_expand "<optab><mode>3"
5118   [(set (match_operand:GPI 0 "register_operand")
5119         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
5120                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
5121   ""
5122   {
5123     if (CONST_INT_P (operands[2]))
5124       {
5125         operands[2] = GEN_INT (INTVAL (operands[2])
5126                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5127
5128         if (operands[2] == const0_rtx)
5129           {
5130             emit_insn (gen_mov<mode> (operands[0], operands[1]));
5131             DONE;
5132           }
5133       }
5134   }
5135 )
5136
5137 (define_expand "ashl<mode>3"
5138   [(set (match_operand:SHORT 0 "register_operand")
5139         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
5140                       (match_operand:QI 2 "const_int_operand")))]
5141   ""
5142   {
5143     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
5144
5145     if (operands[2] == const0_rtx)
5146       {
5147         emit_insn (gen_mov<mode> (operands[0], operands[1]));
5148         DONE;
5149       }
5150   }
5151 )
5152
5153 (define_expand "rotr<mode>3"
5154   [(set (match_operand:GPI 0 "register_operand")
5155         (rotatert:GPI (match_operand:GPI 1 "register_operand")
5156                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
5157   ""
5158   {
5159     if (CONST_INT_P (operands[2]))
5160       {
5161         operands[2] = GEN_INT (INTVAL (operands[2])
5162                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5163
5164         if (operands[2] == const0_rtx)
5165           {
5166             emit_insn (gen_mov<mode> (operands[0], operands[1]));
5167             DONE;
5168           }
5169       }
5170   }
5171 )
5172
5173 (define_expand "rotl<mode>3"
5174   [(set (match_operand:GPI 0 "register_operand")
5175         (rotatert:GPI (match_operand:GPI 1 "register_operand")
5176                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
5177   ""
5178   {
5179     /* (SZ - cnt) % SZ == -cnt % SZ */
5180     if (CONST_INT_P (operands[2]))
5181       {
5182         operands[2] = GEN_INT ((-INTVAL (operands[2]))
5183                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5184         if (operands[2] == const0_rtx)
5185           {
5186             emit_insn (gen_mov<mode> (operands[0], operands[1]));
5187             DONE;
5188           }
5189       }
5190     else
5191       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
5192                                         NULL_RTX, 1);
5193   }
5194 )
5195
5196 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5197 ;; they truncate the shift/rotate amount by the size of the registers they
5198 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
5199 ;; such redundant masking instructions.  GCC can do that automatically when
5200 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5201 ;; because some of the SISD shift alternatives don't perform this truncations.
5202 ;; So this pattern exists to catch such cases.
5203
5204 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5205   [(set (match_operand:GPI 0 "register_operand" "=r")
5206         (SHIFT:GPI
5207           (match_operand:GPI 1 "register_operand" "r")
5208           (match_operator 4 "subreg_lowpart_operator"
5209            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5210                      (match_operand 3 "const_int_operand" "n"))])))]
5211   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5212   "<shift>\t%<w>0, %<w>1, %<w>2"
5213   [(set_attr "type" "shift_reg")]
5214 )
5215
5216 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5217   [(set (match_operand:GPI 0 "register_operand" "=&r")
5218         (SHIFT:GPI
5219           (match_operand:GPI 1 "register_operand" "r")
5220           (match_operator 4 "subreg_lowpart_operator"
5221           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5222                            (match_operand 3 "const_int_operand" "n")))])))]
5223   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5224   "#"
5225   "&& true"
5226   [(const_int 0)]
5227   {
5228     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5229                : lowpart_subreg (SImode, operands[0], <MODE>mode));
5230     emit_insn (gen_negsi2 (tmp, operands[2]));
5231
5232     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5233     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5234                                      SUBREG_BYTE (operands[4]));
5235     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5236     DONE;
5237   }
5238 )
5239
5240 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5241   [(set (match_operand:GPI 0 "register_operand" "=&r")
5242         (ashift:GPI
5243           (match_operand:GPI 1 "register_operand" "r")
5244           (minus:QI (match_operand 2 "const_int_operand" "n")
5245                     (match_operator 5 "subreg_lowpart_operator"
5246                     [(and:SI (match_operand:SI 3 "register_operand" "r")
5247                              (match_operand 4 "const_int_operand" "n"))]))))]
5248   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5249    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5250   "#"
5251   "&& true"
5252   [(const_int 0)]
5253   {
5254     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5255                : operands[0]);
5256
5257     emit_insn (gen_negsi2 (tmp, operands[3]));
5258
5259     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5260     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5261                                      SUBREG_BYTE (operands[5]));
5262
5263     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5264     DONE;
5265   }
5266 )
5267
5268 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5269   [(set (match_operand:DI 0 "register_operand" "=r")
5270         (SHIFT:DI
5271           (match_operand:DI 1 "register_operand" "r")
5272           (match_operator 4 "subreg_lowpart_operator"
5273            [(and:SI (match_operand:SI 2 "register_operand" "r")
5274                     (match_operand 3 "const_int_operand" "n"))])))]
5275   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5276 {
5277   rtx xop[3];
5278   xop[0] = operands[0];
5279   xop[1] = operands[1];
5280   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5281   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5282   return "";
5283 }
5284   [(set_attr "type" "shift_reg")]
5285 )
5286
5287 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5288   [(set (match_operand:GPI 0 "register_operand" "=&r")
5289         (ASHIFT:GPI
5290           (match_operand:GPI 1 "register_operand" "r")
5291           (minus:QI (match_operand 2 "const_int_operand" "n")
5292                     (match_operand:QI 3 "register_operand" "r"))))]
5293   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5294   "#"
5295   "&& true"
5296   [(const_int 0)]
5297   {
5298     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5299
5300     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5301                : gen_lowpart (SImode, operands[0]));
5302
5303     emit_insn (gen_negsi2 (tmp, subreg_tmp));
5304
5305     rtx and_op = gen_rtx_AND (SImode, tmp,
5306                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5307
5308     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5309
5310     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5311     DONE;
5312   }
5313   [(set_attr "length" "8")]
5314 )
5315
5316 ;; Logical left shift using SISD or Integer instruction
5317 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5318   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5319         (ashift:GPI
5320           (match_operand:GPI 1 "register_operand" "r,r,w,w")
5321           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5322   ""
5323   "@
5324    lsl\t%<w>0, %<w>1, %2
5325    lsl\t%<w>0, %<w>1, %<w>2
5326    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5327    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5328   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5329    (set_attr "arch" "*,*,simd,simd")]
5330 )
5331
5332 ;; Logical right shift using SISD or Integer instruction
5333 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5334   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5335         (lshiftrt:GPI
5336          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5337          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5338                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5339   ""
5340   "@
5341    lsr\t%<w>0, %<w>1, %2
5342    lsr\t%<w>0, %<w>1, %<w>2
5343    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5344    #
5345    #"
5346   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5347    (set_attr "arch" "*,*,simd,simd,simd")]
5348 )
5349
5350 (define_split
5351   [(set (match_operand:DI 0 "aarch64_simd_register")
5352         (lshiftrt:DI
5353            (match_operand:DI 1 "aarch64_simd_register")
5354            (match_operand:QI 2 "aarch64_simd_register")))]
5355   "TARGET_SIMD && reload_completed"
5356   [(set (match_dup 3)
5357         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5358    (set (match_dup 0)
5359         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5360   {
5361     operands[3] = gen_lowpart (QImode, operands[0]);
5362   }
5363 )
5364
5365 (define_split
5366   [(set (match_operand:SI 0 "aarch64_simd_register")
5367         (lshiftrt:SI
5368            (match_operand:SI 1 "aarch64_simd_register")
5369            (match_operand:QI 2 "aarch64_simd_register")))]
5370   "TARGET_SIMD && reload_completed"
5371   [(set (match_dup 3)
5372         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5373    (set (match_dup 0)
5374         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5375   {
5376     operands[3] = gen_lowpart (QImode, operands[0]);
5377   }
5378 )
5379
5380 ;; Arithmetic right shift using SISD or Integer instruction
5381 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5382   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5383         (ashiftrt:GPI
5384           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5385           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5386                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5387   ""
5388   "@
5389    asr\t%<w>0, %<w>1, %2
5390    asr\t%<w>0, %<w>1, %<w>2
5391    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5392    #
5393    #"
5394   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5395    (set_attr "arch" "*,*,simd,simd,simd")]
5396 )
5397
5398 (define_split
5399   [(set (match_operand:DI 0 "aarch64_simd_register")
5400         (ashiftrt:DI
5401            (match_operand:DI 1 "aarch64_simd_register")
5402            (match_operand:QI 2 "aarch64_simd_register")))]
5403   "TARGET_SIMD && reload_completed"
5404   [(set (match_dup 3)
5405         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5406    (set (match_dup 0)
5407         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5408 {
5409   operands[3] = gen_lowpart (QImode, operands[0]);
5410 }
5411 )
5412
5413 (define_split
5414   [(set (match_operand:SI 0 "aarch64_simd_register")
5415         (ashiftrt:SI
5416            (match_operand:SI 1 "aarch64_simd_register")
5417            (match_operand:QI 2 "aarch64_simd_register")))]
5418   "TARGET_SIMD && reload_completed"
5419   [(set (match_dup 3)
5420         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5421    (set (match_dup 0)
5422         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5423 {
5424   operands[3] = gen_lowpart (QImode, operands[0]);
5425 }
5426 )
5427
5428 (define_insn "*aarch64_sisd_ushl"
5429   [(set (match_operand:DI 0 "register_operand" "=w")
5430         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5431                     (match_operand:QI 2 "register_operand" "w")]
5432                    UNSPEC_SISD_USHL))]
5433   "TARGET_SIMD"
5434   "ushl\t%d0, %d1, %d2"
5435   [(set_attr "type" "neon_shift_reg")]
5436 )
5437
5438 (define_insn "*aarch64_ushl_2s"
5439   [(set (match_operand:SI 0 "register_operand" "=w")
5440         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5441                     (match_operand:QI 2 "register_operand" "w")]
5442                    UNSPEC_USHL_2S))]
5443   "TARGET_SIMD"
5444   "ushl\t%0.2s, %1.2s, %2.2s"
5445   [(set_attr "type" "neon_shift_reg")]
5446 )
5447
5448 (define_insn "*aarch64_sisd_sshl"
5449   [(set (match_operand:DI 0 "register_operand" "=w")
5450         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5451                     (match_operand:QI 2 "register_operand" "w")]
5452                    UNSPEC_SISD_SSHL))]
5453   "TARGET_SIMD"
5454   "sshl\t%d0, %d1, %d2"
5455   [(set_attr "type" "neon_shift_reg")]
5456 )
5457
5458 (define_insn "*aarch64_sshl_2s"
5459   [(set (match_operand:SI 0 "register_operand" "=w")
5460         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5461                     (match_operand:QI 2 "register_operand" "w")]
5462                    UNSPEC_SSHL_2S))]
5463   "TARGET_SIMD"
5464   "sshl\t%0.2s, %1.2s, %2.2s"
5465   [(set_attr "type" "neon_shift_reg")]
5466 )
5467
5468 (define_insn "*aarch64_sisd_neg_qi"
5469   [(set (match_operand:QI 0 "register_operand" "=w")
5470         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5471                    UNSPEC_SISD_NEG))]
5472   "TARGET_SIMD"
5473   "neg\t%d0, %d1"
5474   [(set_attr "type" "neon_neg")]
5475 )
5476
5477 ;; Rotate right
5478 (define_insn "*ror<mode>3_insn"
5479   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5480      (rotatert:GPI
5481        (match_operand:GPI 1 "register_operand" "r,r")
5482        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5483   ""
5484   "@
5485    ror\\t%<w>0, %<w>1, %2
5486    ror\\t%<w>0, %<w>1, %<w>2"
5487   [(set_attr "type" "rotate_imm,shift_reg")]
5488 )
5489
5490 ;; zero_extend version of above
5491 (define_insn "*<optab>si3_insn_uxtw"
5492   [(set (match_operand:DI 0 "register_operand" "=r,r")
5493         (zero_extend:DI (SHIFT:SI
5494          (match_operand:SI 1 "register_operand" "r,r")
5495          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5496   ""
5497   "@
5498    <shift>\\t%w0, %w1, %2
5499    <shift>\\t%w0, %w1, %w2"
5500   [(set_attr "type" "bfx,shift_reg")]
5501 )
5502
5503 (define_insn "*<optab><mode>3_insn"
5504   [(set (match_operand:SHORT 0 "register_operand" "=r")
5505         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5506                       (match_operand 2 "const_int_operand" "n")))]
5507   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5508 {
5509   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5510   return "<bfshift>\t%w0, %w1, %2, %3";
5511 }
5512   [(set_attr "type" "bfx")]
5513 )
5514
5515 (define_insn "*extr<mode>5_insn"
5516   [(set (match_operand:GPI 0 "register_operand" "=r")
5517         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5518                              (match_operand 3 "const_int_operand" "n"))
5519                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5520                                (match_operand 4 "const_int_operand" "n"))))]
5521   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5522    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5523   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5524   [(set_attr "type" "rotate_imm")]
5525 )
5526
5527 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5528 ;; so we have to match both orderings.
5529 (define_insn "*extr<mode>5_insn_alt"
5530   [(set (match_operand:GPI 0 "register_operand" "=r")
5531         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5532                                 (match_operand 4 "const_int_operand" "n"))
5533                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5534                               (match_operand 3 "const_int_operand" "n"))))]
5535   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5536    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5537        == GET_MODE_BITSIZE (<MODE>mode))"
5538   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5539   [(set_attr "type" "rotate_imm")]
5540 )
5541
5542 ;; zero_extend version of the above
5543 (define_insn "*extrsi5_insn_uxtw"
5544   [(set (match_operand:DI 0 "register_operand" "=r")
5545         (zero_extend:DI
5546          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5547                             (match_operand 3 "const_int_operand" "n"))
5548                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5549                               (match_operand 4 "const_int_operand" "n")))))]
5550   "UINTVAL (operands[3]) < 32 &&
5551    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5552   "extr\\t%w0, %w1, %w2, %4"
5553   [(set_attr "type" "rotate_imm")]
5554 )
5555
5556 (define_insn "*extrsi5_insn_uxtw_alt"
5557   [(set (match_operand:DI 0 "register_operand" "=r")
5558         (zero_extend:DI
5559          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5560                                (match_operand 4 "const_int_operand" "n"))
5561                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5562                             (match_operand 3 "const_int_operand" "n")))))]
5563   "UINTVAL (operands[3]) < 32 &&
5564    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5565   "extr\\t%w0, %w1, %w2, %4"
5566   [(set_attr "type" "rotate_imm")]
5567 )
5568
5569 (define_insn "*ror<mode>3_insn"
5570   [(set (match_operand:GPI 0 "register_operand" "=r")
5571         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5572                     (match_operand 2 "const_int_operand" "n")))]
5573   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5574 {
5575   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5576   return "ror\\t%<w>0, %<w>1, %3";
5577 }
5578   [(set_attr "type" "rotate_imm")]
5579 )
5580
5581 ;; zero_extend version of the above
5582 (define_insn "*rorsi3_insn_uxtw"
5583   [(set (match_operand:DI 0 "register_operand" "=r")
5584         (zero_extend:DI
5585          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5586                     (match_operand 2 "const_int_operand" "n"))))]
5587   "UINTVAL (operands[2]) < 32"
5588 {
5589   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5590   return "ror\\t%w0, %w1, %3";
5591 }
5592   [(set_attr "type" "rotate_imm")]
5593 )
5594
5595 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5596   [(set (match_operand:GPI 0 "register_operand" "=r")
5597         (ANY_EXTEND:GPI
5598          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5599                        (match_operand 2 "const_int_operand" "n"))))]
5600   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5601 {
5602   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5603   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5604 }
5605   [(set_attr "type" "bfx")]
5606 )
5607
5608 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5609   [(set (match_operand:GPI 0 "register_operand" "=r")
5610         (zero_extend:GPI
5611          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5612                          (match_operand 2 "const_int_operand" "n"))))]
5613   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5614 {
5615   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5616   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5617 }
5618   [(set_attr "type" "bfx")]
5619 )
5620
5621 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5622   [(set (match_operand:GPI 0 "register_operand" "=r")
5623         (sign_extend:GPI
5624          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5625                          (match_operand 2 "const_int_operand" "n"))))]
5626   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5627 {
5628   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5629   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5630 }
5631   [(set_attr "type" "bfx")]
5632 )
5633
5634 ;; -------------------------------------------------------------------
5635 ;; Bitfields
5636 ;; -------------------------------------------------------------------
5637
5638 (define_expand "<optab>"
5639   [(set (match_operand:DI 0 "register_operand")
5640         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5641                         (match_operand 2
5642                           "aarch64_simd_shift_imm_offset_di")
5643                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5644   ""
5645   {
5646     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5647                    1, GET_MODE_BITSIZE (DImode) - 1))
5648      FAIL;
5649   }
5650 )
5651
5652
5653 (define_insn "*<optab><mode>"
5654   [(set (match_operand:GPI 0 "register_operand" "=r")
5655         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5656                          (match_operand 2
5657                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5658                          (match_operand 3
5659                            "aarch64_simd_shift_imm_<mode>" "n")))]
5660   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5661              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5662   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5663   [(set_attr "type" "bfx")]
5664 )
5665
5666 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5667 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5668 (define_split
5669   [(set (match_operand:DI 0 "register_operand")
5670         (zero_extract:DI (match_operand:DI 1 "register_operand")
5671                          (match_operand 2
5672                            "aarch64_simd_shift_imm_offset_di")
5673                          (match_operand 3
5674                            "aarch64_simd_shift_imm_di")))]
5675   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5676              GET_MODE_BITSIZE (DImode) - 1)
5677    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5678        == GET_MODE_BITSIZE (SImode)"
5679   [(set (match_dup 0)
5680         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5681   {
5682     operands[4] = gen_lowpart (SImode, operands[1]);
5683   }
5684 )
5685
5686 ;; Bitfield Insert (insv)
5687 (define_expand "insv<mode>"
5688   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5689                           (match_operand 1 "const_int_operand")
5690                           (match_operand 2 "const_int_operand"))
5691         (match_operand:GPI 3 "general_operand"))]
5692   ""
5693 {
5694   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5695   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5696   rtx value = operands[3];
5697
5698   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5699     FAIL;
5700
5701   if (CONST_INT_P (value))
5702     {
5703       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5704
5705       /* Prefer AND/OR for inserting all zeros or all ones.  */
5706       if ((UINTVAL (value) & mask) == 0
5707            || (UINTVAL (value) & mask) == mask)
5708         FAIL;
5709
5710       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5711       if (width == 16 && (pos % 16) == 0)
5712         DONE;
5713     }
5714   operands[3] = force_reg (<MODE>mode, value);
5715 })
5716
5717 (define_insn "*insv_reg<mode>"
5718   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5719                           (match_operand 1 "const_int_operand" "n")
5720                           (match_operand 2 "const_int_operand" "n"))
5721         (match_operand:GPI 3 "register_operand" "r"))]
5722   "!(UINTVAL (operands[1]) == 0
5723      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5724          > GET_MODE_BITSIZE (<MODE>mode)))"
5725   "bfi\\t%<w>0, %<w>3, %2, %1"
5726   [(set_attr "type" "bfm")]
5727 )
5728
5729 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5730   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5731                           (match_operand 1 "const_int_operand" "n")
5732                           (match_operand 2 "const_int_operand" "n"))
5733         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5734   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5735   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5736   [(set_attr "type" "bfm")]
5737 )
5738
5739 ;;  Match a bfi instruction where the shift of OP3 means that we are
5740 ;;  actually copying the least significant bits of OP3 into OP0 by way
5741 ;;  of the AND masks and the IOR instruction.  A similar instruction
5742 ;;  with the two parts of the IOR swapped around was never triggered
5743 ;;  in a bootstrap build and test of GCC so it was not included.
5744
5745 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5746   [(set (match_operand:GPI 0 "register_operand" "=r")
5747         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5748                           (match_operand:GPI 2 "const_int_operand" "n"))
5749                  (and:GPI (ashift:GPI
5750                            (match_operand:GPI 3 "register_operand" "r")
5751                            (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5752                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5753   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5754                                       UINTVAL (operands[4]),
5755                                       UINTVAL(operands[5]))"
5756   "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5757   [(set_attr "type" "bfm")]
5758 )
5759
5760 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5761   [(set (match_operand:GPI 0 "register_operand" "=r")
5762         (ior:GPI (and:GPI (ashift:GPI
5763                            (match_operand:GPI 1 "register_operand" "r")
5764                            (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5765                           (match_operand:GPI 3 "const_int_operand" "n"))
5766                  (and:GPI (match_operand:GPI 4 "register_operand" "0")
5767                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5768   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5769                                       UINTVAL (operands[2]),
5770                                       UINTVAL(operands[3]))"
5771   "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5772   [(set_attr "type" "bfm")]
5773 )
5774
5775 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5776 ;; the shift is large enough to remove the need for an AND instruction.
5777
5778 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5779   [(set (match_operand:GPI 0 "register_operand" "=r")
5780         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5781                           (match_operand:GPI 2 "const_int_operand" "n"))
5782                  (ashift:GPI
5783                           (match_operand:GPI 3 "register_operand" "r")
5784                           (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5785   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5786                                       UINTVAL (operands[4]),
5787                                       HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5788 {
5789   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5790   return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5791 }
5792   [(set_attr "type" "bfm")]
5793 )
5794
5795 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5796   [(set (match_operand:GPI 0 "register_operand" "=r")
5797         (ior:GPI (ashift:GPI
5798                           (match_operand:GPI 1 "register_operand" "r")
5799                           (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5800                  (and:GPI (match_operand:GPI 3 "register_operand" "0")
5801                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5802   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5803                                       UINTVAL (operands[2]),
5804                                       HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5805 {
5806   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5807   return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5808 }
5809   [(set_attr "type" "bfm")]
5810 )
5811
5812 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5813 ;; copying the least significant bits of OP3 to OP0.  We need two versions
5814 ;; of the instruction to handle different checks on the constant values.
5815
5816 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5817   [(set (match_operand:GPI 0 "register_operand" "=r")
5818         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5819                           (match_operand:GPI 2 "const_int_operand" "n"))
5820                  (and:GPI (match_operand:GPI 3 "register_operand" "r")
5821                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5822   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5823                                       UINTVAL (operands[4]))"
5824   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5825   [(set_attr "type" "bfm")]
5826 )
5827
5828 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5829   [(set (match_operand:GPI 0 "register_operand" "=r")
5830         (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5831                           (match_operand:GPI 4 "const_int_operand" "n"))
5832                  (and:GPI (match_operand:GPI 1 "register_operand" "0")
5833                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5834   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5835                                       UINTVAL (operands[4]))"
5836   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5837   [(set_attr "type" "bfm")]
5838 )
5839
5840 (define_insn "*extr_insv_lower_reg<mode>"
5841   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5842                           (match_operand 1 "const_int_operand" "n")
5843                           (const_int 0))
5844         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5845                           (match_dup 1)
5846                           (match_operand 3 "const_int_operand" "n")))]
5847   "!(UINTVAL (operands[1]) == 0
5848      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5849          > GET_MODE_BITSIZE (<MODE>mode)))"
5850   "bfxil\\t%<w>0, %<w>2, %3, %1"
5851   [(set_attr "type" "bfm")]
5852 )
5853
5854 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5855   [(set (match_operand:GPI 0 "register_operand" "=r")
5856         (ashift:GPI (ANY_EXTEND:GPI
5857                      (match_operand:ALLX 1 "register_operand" "r"))
5858                     (match_operand 2 "const_int_operand" "n")))]
5859   "UINTVAL (operands[2]) < <GPI:sizen>"
5860 {
5861   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5862               ? GEN_INT (<ALLX:sizen>)
5863               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5864   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5865 }
5866   [(set_attr "type" "bfx")]
5867 )
5868
5869 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5870
5871 (define_insn "*andim_ashift<mode>_bfiz"
5872   [(set (match_operand:GPI 0 "register_operand" "=r")
5873         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5874                              (match_operand 2 "const_int_operand" "n"))
5875                  (match_operand 3 "const_int_operand" "n")))]
5876   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5877   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5878   [(set_attr "type" "bfx")]
5879 )
5880
5881 ;; Match sbfiz pattern in a shift left + shift right operation.
5882
5883 (define_insn "*ashift<mode>_extv_bfiz"
5884   [(set (match_operand:GPI 0 "register_operand" "=r")
5885         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5886                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5887                                       (const_int 0))
5888                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5889   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5890              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5891   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5892   [(set_attr "type" "bfx")]
5893 )
5894
5895 (define_insn "*ashiftsi_extvdi_bfiz"
5896   [(set (match_operand:SI 0 "register_operand" "=r")
5897         (ashift:SI
5898           (match_operator:SI 4 "subreg_lowpart_operator"
5899             [(sign_extract:DI
5900                (match_operand:DI 1 "register_operand" "r")
5901                (match_operand 2 "aarch64_simd_shift_imm_offset_si")
5902                (const_int 0))])
5903           (match_operand 3 "aarch64_simd_shift_imm_si")))]
5904   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5905              1, GET_MODE_BITSIZE (SImode) - 1)"
5906   "sbfiz\\t%w0, %w1, %3, %2"
5907   [(set_attr "type" "bfx")]
5908 )
5909
5910 ;; When the bit position and width of the equivalent extraction add up to 32
5911 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5912 ;; zero-extension of the X-reg.
5913 (define_split
5914   [(set (match_operand:DI 0 "register_operand")
5915         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5916                              (match_operand 2 "const_int_operand"))
5917                  (match_operand 3 "const_int_operand")))]
5918  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5919   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5920       == GET_MODE_BITSIZE (SImode)"
5921   [(set (match_dup 0)
5922         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5923   {
5924     operands[4] = gen_lowpart (SImode, operands[1]);
5925   }
5926 )
5927
5928 (define_insn "bswap<mode>2"
5929   [(set (match_operand:GPI 0 "register_operand" "=r")
5930         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5931   ""
5932   "rev\\t%<w>0, %<w>1"
5933   [(set_attr "type" "rev")]
5934 )
5935
5936 (define_insn "bswaphi2"
5937   [(set (match_operand:HI 0 "register_operand" "=r")
5938         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5939   ""
5940   "rev16\\t%w0, %w1"
5941   [(set_attr "type" "rev")]
5942 )
5943
5944 (define_insn "*aarch64_bfxil<mode>"
5945   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5946     (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5947                     (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5948             (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5949                     (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5950   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5951   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5952     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5953   {
5954     switch (which_alternative)
5955     {
5956       case 0:
5957         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5958         return "bfxil\\t%<w>0, %<w>1, 0, %3";
5959       case 1:
5960         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5961         return "bfxil\\t%<w>0, %<w>2, 0, %3";
5962       default:
5963         gcc_unreachable ();
5964     }
5965   }
5966   [(set_attr "type" "bfm")]
5967 )
5968
5969 ; Zero-extended version of above (aarch64_bfxil)
5970 (define_insn "*aarch64_bfxilsi_uxtw"
5971   [(set (match_operand:DI 0 "register_operand" "=r,r")
5972         (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5973                                         "r,0")
5974                     (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5975             (and:SI (match_operand:SI 2 "register_operand" "0,r")
5976                     (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5977   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5978   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5979     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5980   {
5981     switch (which_alternative)
5982     {
5983       case 0:
5984         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5985         return "bfxil\\t%0, %1, 0, %3";
5986       case 1:
5987         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5988         return "bfxil\\t%0, %2, 0, %3";
5989       default:
5990         gcc_unreachable ();
5991     }
5992   }
5993   [(set_attr "type" "bfm")]
5994 )
5995
5996 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5997 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5998 ;; each valid permutation.
5999
6000 (define_insn "rev16<mode>2"
6001   [(set (match_operand:GPI 0 "register_operand" "=r")
6002         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
6003                                       (const_int 8))
6004                           (match_operand:GPI 3 "const_int_operand" "n"))
6005                  (and:GPI (lshiftrt:GPI (match_dup 1)
6006                                         (const_int 8))
6007                           (match_operand:GPI 2 "const_int_operand" "n"))))]
6008   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
6009    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
6010   "rev16\\t%<w>0, %<w>1"
6011   [(set_attr "type" "rev")]
6012 )
6013
6014 (define_insn "rev16<mode>2_alt"
6015   [(set (match_operand:GPI 0 "register_operand" "=r")
6016         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
6017                                         (const_int 8))
6018                           (match_operand:GPI 2 "const_int_operand" "n"))
6019                  (and:GPI (ashift:GPI (match_dup 1)
6020                                       (const_int 8))
6021                           (match_operand:GPI 3 "const_int_operand" "n"))))]
6022   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
6023    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
6024   "rev16\\t%<w>0, %<w>1"
6025   [(set_attr "type" "rev")]
6026 )
6027
6028 ;; zero_extend version of above
6029 (define_insn "*bswapsi2_uxtw"
6030   [(set (match_operand:DI 0 "register_operand" "=r")
6031         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
6032   ""
6033   "rev\\t%w0, %w1"
6034   [(set_attr "type" "rev")]
6035 )
6036
6037 ;; -------------------------------------------------------------------
6038 ;; Floating-point intrinsics
6039 ;; -------------------------------------------------------------------
6040
6041 ;; frint floating-point round to integral standard patterns.
6042 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
6043
6044 (define_insn "<frint_pattern><mode>2"
6045   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6046         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
6047          FRINT))]
6048   "TARGET_FLOAT"
6049   "frint<frint_suffix>\\t%<s>0, %<s>1"
6050   [(set_attr "type" "f_rint<stype>")]
6051 )
6052
6053 ;; frcvt floating-point round to integer and convert standard patterns.
6054 ;; Expands to lbtrunc, lceil, lfloor, lround.
6055 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
6056   [(set (match_operand:GPI 0 "register_operand" "=r")
6057         (FIXUORS:GPI
6058           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
6059            FCVT)))]
6060   "TARGET_FLOAT"
6061   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
6062   [(set_attr "type" "f_cvtf2i")]
6063 )
6064
6065 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
6066   [(set (match_operand:GPI 0 "register_operand" "=r")
6067         (FIXUORS:GPI
6068           (mult:GPF
6069             (match_operand:GPF 1 "register_operand" "w")
6070             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
6071   "TARGET_FLOAT
6072    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
6073                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
6074   {
6075     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
6076     char buf[64];
6077     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
6078     output_asm_insn (buf, operands);
6079     return "";
6080   }
6081   [(set_attr "type" "f_cvtf2i")]
6082 )
6083
6084 ;; fma - expand fma into patterns with the accumulator operand first since
6085 ;; reusing the accumulator results in better register allocation.
6086 ;; The register allocator considers copy preferences in operand order,
6087 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
6088
6089 (define_expand "fma<mode>4"
6090   [(set (match_operand:GPF_F16 0 "register_operand")
6091         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
6092                      (match_operand:GPF_F16 2 "register_operand")
6093                      (match_operand:GPF_F16 3 "register_operand")))]
6094   "TARGET_FLOAT"
6095 )
6096
6097 (define_insn "*aarch64_fma<mode>4"
6098   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6099         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
6100                      (match_operand:GPF_F16 3 "register_operand" "w")
6101                      (match_operand:GPF_F16 1 "register_operand" "w")))]
6102   "TARGET_FLOAT"
6103   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
6104   [(set_attr "type" "fmac<stype>")]
6105 )
6106
6107 (define_expand "fnma<mode>4"
6108   [(set (match_operand:GPF_F16 0 "register_operand")
6109         (fma:GPF_F16
6110           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
6111           (match_operand:GPF_F16 2 "register_operand")
6112           (match_operand:GPF_F16 3 "register_operand")))]
6113   "TARGET_FLOAT"
6114 )
6115
6116 (define_insn "*aarch64_fnma<mode>4"
6117   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6118         (fma:GPF_F16
6119           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
6120           (match_operand:GPF_F16 3 "register_operand" "w")
6121           (match_operand:GPF_F16 1 "register_operand" "w")))]
6122   "TARGET_FLOAT"
6123   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
6124   [(set_attr "type" "fmac<stype>")]
6125 )
6126
6127
6128 (define_expand "fms<mode>4"
6129   [(set (match_operand:GPF 0 "register_operand")
6130         (fma:GPF (match_operand:GPF 1 "register_operand")
6131                  (match_operand:GPF 2 "register_operand")
6132                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
6133   "TARGET_FLOAT"
6134 )
6135
6136 (define_insn "*aarch64_fms<mode>4"
6137   [(set (match_operand:GPF 0 "register_operand" "=w")
6138         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
6139                  (match_operand:GPF 3 "register_operand" "w")
6140                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
6141   "TARGET_FLOAT"
6142   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
6143   [(set_attr "type" "fmac<s>")]
6144 )
6145
6146 (define_expand "fnms<mode>4"
6147   [(set (match_operand:GPF 0 "register_operand")
6148         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
6149                  (match_operand:GPF 2 "register_operand")
6150                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
6151   "TARGET_FLOAT"
6152 )
6153
6154 (define_insn "*aarch64_fnms<mode>4"
6155   [(set (match_operand:GPF 0 "register_operand" "=w")
6156         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
6157                  (match_operand:GPF 3 "register_operand" "w")
6158                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
6159   "TARGET_FLOAT"
6160   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
6161   [(set_attr "type" "fmac<s>")]
6162 )
6163
6164 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
6165 (define_insn "*aarch64_fnmadd<mode>4"
6166   [(set (match_operand:GPF 0 "register_operand" "=w")
6167         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
6168                           (match_operand:GPF 3 "register_operand" "w")
6169                           (match_operand:GPF 1 "register_operand" "w"))))]
6170   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
6171   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
6172   [(set_attr "type" "fmac<s>")]
6173 )
6174
6175 ;; -------------------------------------------------------------------
6176 ;; Floating-point conversions
6177 ;; -------------------------------------------------------------------
6178
6179 (define_insn "extendsfdf2"
6180   [(set (match_operand:DF 0 "register_operand" "=w")
6181         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
6182   "TARGET_FLOAT"
6183   "fcvt\\t%d0, %s1"
6184   [(set_attr "type" "f_cvt")]
6185 )
6186
6187 (define_insn "extendhfsf2"
6188   [(set (match_operand:SF 0 "register_operand" "=w")
6189         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
6190   "TARGET_FLOAT"
6191   "fcvt\\t%s0, %h1"
6192   [(set_attr "type" "f_cvt")]
6193 )
6194
6195 (define_insn "extendhfdf2"
6196   [(set (match_operand:DF 0 "register_operand" "=w")
6197         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
6198   "TARGET_FLOAT"
6199   "fcvt\\t%d0, %h1"
6200   [(set_attr "type" "f_cvt")]
6201 )
6202
6203 (define_insn "truncdfsf2"
6204   [(set (match_operand:SF 0 "register_operand" "=w")
6205         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
6206   "TARGET_FLOAT"
6207   "fcvt\\t%s0, %d1"
6208   [(set_attr "type" "f_cvt")]
6209 )
6210
6211 (define_insn "truncsfhf2"
6212   [(set (match_operand:HF 0 "register_operand" "=w")
6213         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6214   "TARGET_FLOAT"
6215   "fcvt\\t%h0, %s1"
6216   [(set_attr "type" "f_cvt")]
6217 )
6218
6219 (define_insn "truncdfhf2"
6220   [(set (match_operand:HF 0 "register_operand" "=w")
6221         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6222   "TARGET_FLOAT"
6223   "fcvt\\t%h0, %d1"
6224   [(set_attr "type" "f_cvt")]
6225 )
6226
6227 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6228 ;; and making r = w more expensive
6229
6230 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
6231   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
6232         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6233   "TARGET_FLOAT"
6234   "@
6235    fcvtz<su>\t%<s>0, %<s>1
6236    fcvtz<su>\t%<w>0, %<s>1"
6237   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
6238 )
6239
6240 ;; Convert HF -> SI or DI
6241
6242 (define_insn "<optab>_trunchf<GPI:mode>2"
6243   [(set (match_operand:GPI 0 "register_operand" "=r")
6244         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6245   "TARGET_FP_F16INST"
6246   "fcvtz<su>\t%<w>0, %h1"
6247   [(set_attr "type" "f_cvtf2i")]
6248 )
6249
6250 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6251 ;; input in a fp register and output in a integer register
6252
6253 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6254   [(set (match_operand:GPI 0 "register_operand" "=r")
6255         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6256   "TARGET_FLOAT"
6257   "fcvtz<su>\t%<w>0, %<fpw>1"
6258   [(set_attr "type" "f_cvtf2i")]
6259 )
6260
6261 (define_insn "*fix_to_zero_extend<mode>di2"
6262   [(set (match_operand:DI 0 "register_operand" "=r")
6263         (zero_extend:DI
6264          (unsigned_fix:SI
6265           (match_operand:GPF 1 "register_operand" "w"))))]
6266   "TARGET_FLOAT"
6267   "fcvtzu\t%w0, %<s>1"
6268   [(set_attr "type" "f_cvtf2i")]
6269 )
6270
6271 ;; Equal width integer to fp and multiply combine.
6272 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6273   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6274         (mult:GPF (FLOATUORS:GPF
6275                    (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6276                    (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6277   "TARGET_FLOAT"
6278   {
6279     operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6280     switch (which_alternative)
6281     {
6282       case 0:
6283         return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6284       case 1:
6285         return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6286       default:
6287         gcc_unreachable ();
6288     }
6289   }
6290   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6291    (set_attr "arch" "simd,fp")]
6292 )
6293
6294 ;; Unequal width integer to fp and multiply combine.
6295 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6296   [(set (match_operand:GPF 0 "register_operand" "=w")
6297         (mult:GPF (FLOATUORS:GPF
6298                    (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6299                    (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6300   "TARGET_FLOAT"
6301   {
6302     operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6303     return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6304   }
6305   [(set_attr "type" "f_cvti2f")]
6306 )
6307
6308 ;; Equal width integer to fp conversion.
6309 (define_insn "<optab><fcvt_target><GPF:mode>2"
6310   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6311         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6312   "TARGET_FLOAT"
6313   "@
6314    <su_optab>cvtf\t%<GPF:s>0, %<s>1
6315    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6316   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6317    (set_attr "arch" "simd,fp")]
6318 )
6319
6320 ;; Unequal width integer to fp conversions.
6321 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6322   [(set (match_operand:GPF 0 "register_operand" "=w")
6323         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6324   "TARGET_FLOAT"
6325   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6326   [(set_attr "type" "f_cvti2f")]
6327 )
6328
6329 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6330 ;; midend will arrange for an SImode conversion to HFmode to first go
6331 ;; through DFmode, then to HFmode.  But first it will try converting
6332 ;; to DImode then down, which would match our DImode pattern below and
6333 ;; give very poor code-generation.  So, we must provide our own emulation
6334 ;; of the mid-end logic.
6335
6336 (define_insn "aarch64_fp16_<optab><mode>hf2"
6337   [(set (match_operand:HF 0 "register_operand" "=w")
6338         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6339   "TARGET_FP_F16INST"
6340   "<su_optab>cvtf\t%h0, %<w>1"
6341   [(set_attr "type" "f_cvti2f")]
6342 )
6343
6344 (define_expand "<optab>sihf2"
6345   [(set (match_operand:HF 0 "register_operand")
6346         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6347   "TARGET_FLOAT"
6348 {
6349   if (TARGET_FP_F16INST)
6350     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6351   else
6352     {
6353       rtx convert_target = gen_reg_rtx (DFmode);
6354       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6355       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6356     }
6357   DONE;
6358 }
6359 )
6360
6361 ;; For DImode there is no wide enough floating-point mode that we
6362 ;; can convert through natively (TFmode would work, but requires a library
6363 ;; call).  However, we know that any value >= 65504 will be rounded
6364 ;; to infinity on conversion.  This is well within the range of SImode, so
6365 ;; we can:
6366 ;;   Saturate to SImode.
6367 ;;   Convert from that to DFmode
6368 ;;   Convert from that to HFmode (phew!).
6369 ;; Note that the saturation to SImode requires the SIMD extensions.  If
6370 ;; we ever need to provide this pattern where the SIMD extensions are not
6371 ;; available, we would need a different approach.
6372
6373 (define_expand "<optab>dihf2"
6374   [(set (match_operand:HF 0 "register_operand")
6375         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6376   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6377 {
6378   if (TARGET_FP_F16INST)
6379     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6380   else
6381     {
6382       rtx sat_target = gen_reg_rtx (SImode);
6383       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6384       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6385     }
6386
6387   DONE;
6388 }
6389 )
6390
6391 ;; Convert between fixed-point and floating-point (scalar modes)
6392
6393 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6394   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6395         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6396                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6397          FCVT_F2FIXED))]
6398   ""
6399   "@
6400    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6401    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6402   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6403    (set_attr "arch" "fp,simd")]
6404 )
6405
6406 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6407   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6408         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6409                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6410          FCVT_FIXED2F))]
6411   ""
6412   "@
6413    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6414    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6415   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6416    (set_attr "arch" "fp,simd")]
6417 )
6418
6419 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6420   [(set (match_operand:GPI 0 "register_operand" "=r")
6421         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6422                      (match_operand:SI 2 "immediate_operand" "i")]
6423          FCVT_F2FIXED))]
6424   "TARGET_FP_F16INST"
6425    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6426   [(set_attr "type" "f_cvtf2i")]
6427 )
6428
6429 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6430   [(set (match_operand:HF 0 "register_operand" "=w")
6431         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6432                     (match_operand:SI 2 "immediate_operand" "i")]
6433          FCVT_FIXED2F))]
6434   "TARGET_FP_F16INST"
6435   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6436   [(set_attr "type" "f_cvti2f")]
6437 )
6438
6439 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6440   [(set (match_operand:HI 0 "register_operand" "=w")
6441         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6442                     (match_operand:SI 2 "immediate_operand" "i")]
6443          FCVT_F2FIXED))]
6444   "TARGET_SIMD"
6445   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6446   [(set_attr "type" "neon_fp_to_int_s")]
6447 )
6448
6449 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6450   [(set (match_operand:HF 0 "register_operand" "=w")
6451         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6452                     (match_operand:SI 2 "immediate_operand" "i")]
6453          FCVT_FIXED2F))]
6454   "TARGET_SIMD"
6455   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6456   [(set_attr "type" "neon_int_to_fp_s")]
6457 )
6458
6459 ;; -------------------------------------------------------------------
6460 ;; Floating-point arithmetic
6461 ;; -------------------------------------------------------------------
6462
6463 (define_insn "add<mode>3"
6464   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6465         (plus:GPF_F16
6466          (match_operand:GPF_F16 1 "register_operand" "w")
6467          (match_operand:GPF_F16 2 "register_operand" "w")))]
6468   "TARGET_FLOAT"
6469   "fadd\\t%<s>0, %<s>1, %<s>2"
6470   [(set_attr "type" "fadd<stype>")]
6471 )
6472
6473 (define_insn "sub<mode>3"
6474   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6475         (minus:GPF_F16
6476          (match_operand:GPF_F16 1 "register_operand" "w")
6477          (match_operand:GPF_F16 2 "register_operand" "w")))]
6478   "TARGET_FLOAT"
6479   "fsub\\t%<s>0, %<s>1, %<s>2"
6480   [(set_attr "type" "fadd<stype>")]
6481 )
6482
6483 (define_insn "mul<mode>3"
6484   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6485         (mult:GPF_F16
6486          (match_operand:GPF_F16 1 "register_operand" "w")
6487          (match_operand:GPF_F16 2 "register_operand" "w")))]
6488   "TARGET_FLOAT"
6489   "fmul\\t%<s>0, %<s>1, %<s>2"
6490   [(set_attr "type" "fmul<stype>")]
6491 )
6492
6493 (define_insn "*fnmul<mode>3"
6494   [(set (match_operand:GPF 0 "register_operand" "=w")
6495         (mult:GPF
6496                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6497                  (match_operand:GPF 2 "register_operand" "w")))]
6498   "TARGET_FLOAT && !flag_rounding_math"
6499   "fnmul\\t%<s>0, %<s>1, %<s>2"
6500   [(set_attr "type" "fmul<s>")]
6501 )
6502
6503 (define_insn "*fnmul<mode>3"
6504   [(set (match_operand:GPF 0 "register_operand" "=w")
6505         (neg:GPF (mult:GPF
6506                  (match_operand:GPF 1 "register_operand" "w")
6507                  (match_operand:GPF 2 "register_operand" "w"))))]
6508   "TARGET_FLOAT"
6509   "fnmul\\t%<s>0, %<s>1, %<s>2"
6510   [(set_attr "type" "fmul<s>")]
6511 )
6512
6513 (define_expand "div<mode>3"
6514  [(set (match_operand:GPF_F16 0 "register_operand")
6515        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6516                     (match_operand:GPF_F16 2 "register_operand")))]
6517  "TARGET_FLOAT"
6518 {
6519   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6520     DONE;
6521
6522   operands[1] = force_reg (<MODE>mode, operands[1]);
6523 })
6524
6525 (define_insn "*div<mode>3"
6526   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6527         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6528                      (match_operand:GPF_F16 2 "register_operand" "w")))]
6529   "TARGET_FLOAT"
6530   "fdiv\\t%<s>0, %<s>1, %<s>2"
6531   [(set_attr "type" "fdiv<stype>")]
6532 )
6533
6534 (define_insn "neg<mode>2"
6535   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6536         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6537   "TARGET_FLOAT"
6538   "fneg\\t%<s>0, %<s>1"
6539   [(set_attr "type" "ffarith<stype>")]
6540 )
6541
6542 (define_expand "sqrt<mode>2"
6543   [(set (match_operand:GPF_F16 0 "register_operand")
6544         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6545   "TARGET_FLOAT"
6546 {
6547   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6548     DONE;
6549 })
6550
6551 (define_insn "*sqrt<mode>2"
6552   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6553         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6554   "TARGET_FLOAT"
6555   "fsqrt\\t%<s>0, %<s>1"
6556   [(set_attr "type" "fsqrt<stype>")]
6557 )
6558
6559 (define_insn "abs<mode>2"
6560   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6561         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6562   "TARGET_FLOAT"
6563   "fabs\\t%<s>0, %<s>1"
6564   [(set_attr "type" "ffarith<stype>")]
6565 )
6566
6567 ;; Given that smax/smin do not specify the result when either input is NaN,
6568 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6569 ;; for smin.
6570
6571 (define_insn "smax<mode>3"
6572   [(set (match_operand:GPF 0 "register_operand" "=w")
6573         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6574                   (match_operand:GPF 2 "register_operand" "w")))]
6575   "TARGET_FLOAT"
6576   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6577   [(set_attr "type" "f_minmax<s>")]
6578 )
6579
6580 (define_insn "smin<mode>3"
6581   [(set (match_operand:GPF 0 "register_operand" "=w")
6582         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6583                   (match_operand:GPF 2 "register_operand" "w")))]
6584   "TARGET_FLOAT"
6585   "fminnm\\t%<s>0, %<s>1, %<s>2"
6586   [(set_attr "type" "f_minmax<s>")]
6587 )
6588
6589 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6590 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6591 ;; which implement the IEEE fmax ()/fmin () functions.
6592 (define_insn "<maxmin_uns><mode>3"
6593   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6594         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6595                      (match_operand:GPF_F16 2 "register_operand" "w")]
6596                      FMAXMIN_UNS))]
6597   "TARGET_FLOAT"
6598   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6599   [(set_attr "type" "f_minmax<stype>")]
6600 )
6601
6602 (define_expand "lrint<GPF:mode><GPI:mode>2"
6603   [(match_operand:GPI 0 "register_operand")
6604    (match_operand:GPF 1 "register_operand")]
6605   "TARGET_FLOAT
6606    && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
6607    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6608 {
6609   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6610   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6611   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6612   DONE;
6613 }
6614 )
6615
6616 ;; For copysign (x, y), we want to generate:
6617 ;;
6618 ;;   LDR d2, #(1 << 63)
6619 ;;   BSL v2.8b, [y], [x]
6620 ;;
6621 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.  Because
6622 ;; we expect these operations to nearly always operate on
6623 ;; floating-point values, we do not want the operation to be
6624 ;; simplified into a bit-field insert operation that operates on the
6625 ;; integer side, since typically that would involve three inter-bank
6626 ;; register copies.  As we do not expect copysign to be followed by
6627 ;; other logical operations on the result, it seems preferable to keep
6628 ;; this as an unspec operation, rather than exposing the underlying
6629 ;; logic to the compiler.
6630
6631 (define_expand "copysign<GPF:mode>3"
6632   [(match_operand:GPF 0 "register_operand")
6633    (match_operand:GPF 1 "register_operand")
6634    (match_operand:GPF 2 "register_operand")]
6635   "TARGET_FLOAT && TARGET_SIMD"
6636 {
6637   rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6638   emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6639                                     << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6640   emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6641                                        bitmask));
6642   DONE;
6643 }
6644 )
6645
6646 (define_insn "copysign<GPF:mode>3_insn"
6647   [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6648         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6649                      (match_operand:GPF 2 "register_operand" "w,w,0,0")
6650                      (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6651          UNSPEC_COPYSIGN))]
6652   "TARGET_FLOAT && TARGET_SIMD"
6653   "@
6654    bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6655    bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6656    bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6657    bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6658   [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6659 )
6660
6661
6662 ;; For xorsign (x, y), we want to generate:
6663 ;;
6664 ;; LDR   d2, #1<<63
6665 ;; AND   v3.8B, v1.8B, v2.8B
6666 ;; EOR   v0.8B, v0.8B, v3.8B
6667 ;;
6668
6669 (define_expand "xorsign<mode>3"
6670   [(match_operand:GPF 0 "register_operand")
6671    (match_operand:GPF 1 "register_operand")
6672    (match_operand:GPF 2 "register_operand")]
6673   "TARGET_FLOAT && TARGET_SIMD"
6674 {
6675
6676   machine_mode imode = <V_INT_EQUIV>mode;
6677   rtx mask = gen_reg_rtx (imode);
6678   rtx op1x = gen_reg_rtx (imode);
6679   rtx op2x = gen_reg_rtx (imode);
6680
6681   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6682   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6683                                                      imode)));
6684
6685   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6686                                     lowpart_subreg (imode, operands[2],
6687                                                     <MODE>mode)));
6688   emit_insn (gen_xor<v_int_equiv>3 (op1x,
6689                                     lowpart_subreg (imode, operands[1],
6690                                                     <MODE>mode),
6691                                     op2x));
6692   emit_move_insn (operands[0],
6693                   lowpart_subreg (<MODE>mode, op1x, imode));
6694   DONE;
6695 }
6696 )
6697
6698 ;; -------------------------------------------------------------------
6699 ;; Reload support
6700 ;; -------------------------------------------------------------------
6701 ;; Reload Scalar Floating point modes from constant pool.
6702 ;; The AArch64 port doesn't have __int128 constant move support.
6703 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6704 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6705  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6706        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6707   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6708  "TARGET_FLOAT"
6709  {
6710    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6711    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6712    DONE;
6713  }
6714 )
6715
6716 ;; Reload Vector modes from constant pool.
6717 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6718  [(set (match_operand:VALL 0 "register_operand" "=w")
6719        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6720   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6721  "TARGET_FLOAT"
6722  {
6723    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6724    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6725    DONE;
6726  }
6727 )
6728
6729 (define_expand "@aarch64_reload_mov<mode>"
6730   [(set (match_operand:TX 0 "register_operand" "=w")
6731         (match_operand:TX 1 "register_operand" "w"))
6732    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6733   ]
6734   "TARGET_FLOAT"
6735   {
6736     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6737     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6738     gen_aarch64_movtilow_tilow (op0, op1);
6739     gen_aarch64_movdi_tihigh (operands[2], op1);
6740     gen_aarch64_movtihigh_di (op0, operands[2]);
6741     DONE;
6742   }
6743 )
6744
6745 ;; The following secondary reload helpers patterns are invoked
6746 ;; after or during reload as we don't want these patterns to start
6747 ;; kicking in during the combiner.
6748
6749 (define_insn "@aarch64_movdi_<mode>low"
6750   [(set (match_operand:DI 0 "register_operand" "=r")
6751         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6752                          (const_int 64) (const_int 0)))]
6753   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6754   "fmov\\t%x0, %d1"
6755   [(set_attr "type" "f_mrc")
6756    (set_attr "length" "4")
6757   ])
6758
6759 (define_insn "@aarch64_movdi_<mode>high"
6760   [(set (match_operand:DI 0 "register_operand" "=r")
6761         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6762                          (const_int 64) (const_int 64)))]
6763   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6764   "fmov\\t%x0, %1.d[1]"
6765   [(set_attr "type" "f_mrc")
6766    (set_attr "length" "4")
6767   ])
6768
6769 (define_insn "@aarch64_mov<mode>high_di"
6770   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6771                          (const_int 64) (const_int 64))
6772         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6773   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6774   "fmov\\t%0.d[1], %x1"
6775   [(set_attr "type" "f_mcr")
6776    (set_attr "length" "4")
6777   ])
6778
6779 (define_insn "@aarch64_mov<mode>low_di"
6780   [(set (match_operand:TX 0 "register_operand" "=w")
6781         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6782   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6783   "fmov\\t%d0, %x1"
6784   [(set_attr "type" "f_mcr")
6785    (set_attr "length" "4")
6786   ])
6787
6788 (define_insn "aarch64_movtilow_tilow"
6789   [(set (match_operand:TI 0 "register_operand" "=w")
6790         (zero_extend:TI
6791           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6792   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6793   "fmov\\t%d0, %d1"
6794   [(set_attr "type" "fmov")
6795    (set_attr "length" "4")
6796   ])
6797
6798 ;; There is a deliberate reason why the parameters of high and lo_sum's
6799 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6800 ;; and lo_sum's to be used with the labels defining the jump tables in
6801 ;; rodata section.
6802
6803 (define_expand "add_losym"
6804   [(set (match_operand 0 "register_operand")
6805         (lo_sum (match_operand 1 "register_operand")
6806                 (match_operand 2 "aarch64_valid_symref")))]
6807   ""
6808 {
6809   machine_mode mode = GET_MODE (operands[0]);
6810   emit_insn (gen_add_losym (mode, operands[0], operands[1], operands[2]));
6811   DONE;
6812 })
6813
6814 (define_insn "@add_losym_<mode>"
6815   [(set (match_operand:P 0 "register_operand" "=r")
6816         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6817                   (match_operand 2 "aarch64_valid_symref" "S")))]
6818   ""
6819   "add\\t%<w>0, %<w>1, :lo12:%c2"
6820   [(set_attr "type" "alu_imm")]
6821 )
6822
6823 (define_insn "@ldr_got_small_<mode>"
6824   [(set (match_operand:PTR 0 "register_operand" "=r")
6825         (unspec:PTR [(mem:PTR (lo_sum:PTR
6826                               (match_operand:PTR 1 "register_operand" "r")
6827                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6828                     UNSPEC_GOTSMALLPIC))]
6829   ""
6830   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6831   [(set_attr "type" "load_<ldst_sz>")]
6832 )
6833
6834 (define_insn "ldr_got_small_sidi"
6835   [(set (match_operand:DI 0 "register_operand" "=r")
6836         (zero_extend:DI
6837          (unspec:SI [(mem:SI (lo_sum:DI
6838                              (match_operand:DI 1 "register_operand" "r")
6839                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6840                     UNSPEC_GOTSMALLPIC)))]
6841   "TARGET_ILP32"
6842   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6843   [(set_attr "type" "load_4")]
6844 )
6845
6846 (define_insn "@ldr_got_small_28k_<mode>"
6847   [(set (match_operand:PTR 0 "register_operand" "=r")
6848         (unspec:PTR [(mem:PTR (lo_sum:PTR
6849                               (match_operand:PTR 1 "register_operand" "r")
6850                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6851                     UNSPEC_GOTSMALLPIC28K))]
6852   ""
6853   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6854   [(set_attr "type" "load_<ldst_sz>")]
6855 )
6856
6857 (define_insn "ldr_got_small_28k_sidi"
6858   [(set (match_operand:DI 0 "register_operand" "=r")
6859         (zero_extend:DI
6860          (unspec:SI [(mem:SI (lo_sum:DI
6861                              (match_operand:DI 1 "register_operand" "r")
6862                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6863                     UNSPEC_GOTSMALLPIC28K)))]
6864   "TARGET_ILP32"
6865   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6866   [(set_attr "type" "load_4")]
6867 )
6868
6869 (define_insn "@ldr_got_tiny_<mode>"
6870   [(set (match_operand:PTR 0 "register_operand" "=r")
6871         (unspec:PTR [(match_operand:PTR 1 "aarch64_valid_symref" "S")]
6872                     UNSPEC_GOTTINYPIC))]
6873   ""
6874   "ldr\t%<w>0, %L1"
6875   [(set_attr "type" "load_<ldst_sz>")]
6876 )
6877
6878 (define_insn "ldr_got_tiny_sidi"
6879   [(set (match_operand:DI 0 "register_operand" "=r")
6880         (zero_extend:DI
6881           (unspec:SI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6882                      UNSPEC_GOTTINYPIC)))]
6883   "TARGET_ILP32"
6884   "ldr\t%w0, %L1"
6885   [(set_attr "type" "load_4")]
6886 )
6887
6888 (define_insn "@aarch64_load_tp_hard_<mode>"
6889   [(set (match_operand:ADDR 0 "register_operand" "=r")
6890         (unspec:ADDR [(const_int 0)] UNSPEC_TLS))]
6891   ""
6892   "mrs\\t%0, tpidr_el0"
6893   [(set_attr "type" "mrs")]
6894 )
6895
6896 ;; The TLS ABI specifically requires that the compiler does not schedule
6897 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6898 ;; Therefore we treat the stubs as an atomic sequence.
6899 (define_expand "tlsgd_small_<mode>"
6900  [(parallel [(set (match_operand:PTR 0 "register_operand")
6901                   (call (mem:DI (match_dup 2)) (const_int 1)))
6902              (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6903              (unspec:DI [(match_operand 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6904              (clobber (reg:DI LR_REGNUM))])]
6905  ""
6906 {
6907   operands[2] = aarch64_tls_get_addr ();
6908 })
6909
6910 (define_insn "*tlsgd_small_<mode>"
6911   [(set (match_operand:PTR 0 "register_operand" "")
6912         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6913    (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6914    (unspec:DI [(match_operand 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6915    (clobber (reg:DI LR_REGNUM))
6916   ]
6917   ""
6918   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6919   [(set_attr "type" "call")
6920    (set_attr "length" "16")])
6921
6922 (define_insn "@tlsie_small_<mode>"
6923   [(set (match_operand:<PTR_OFF> 0 "register_operand" "=r")
6924         (unspec:<PTR_OFF>
6925           [(match_operand 1 "aarch64_tls_ie_symref" "S")
6926            (match_operand:PTR 2 "register_operand" "r")]
6927                    UNSPEC_GOTSMALLTLS))]
6928   "aarch64_lowpart_regs_p (operands[0], operands[2])"
6929   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6930   [(set_attr "type" "load_4")
6931    (set_attr "length" "8")]
6932 )
6933
6934 (define_insn "tlsie_small_sidi"
6935   [(set (match_operand:DI 0 "register_operand" "=r")
6936         (zero_extend:DI
6937           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6938                       UNSPEC_GOTSMALLTLS)))]
6939   ""
6940   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6941   [(set_attr "type" "load_4")
6942    (set_attr "length" "8")]
6943 )
6944
6945 (define_insn "@tlsie_tiny_<mode>"
6946   [(set (match_operand:PTR 0 "register_operand" "=&r")
6947         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6948                      (match_operand:PTR 2 "register_operand" "r")]
6949                    UNSPEC_GOTTINYTLS))]
6950   ""
6951   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6952   [(set_attr "type" "multiple")
6953    (set_attr "length" "8")]
6954 )
6955
6956 (define_insn "tlsie_tiny_sidi"
6957   [(set (match_operand:DI 0 "register_operand" "=&r")
6958         (zero_extend:DI
6959           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6960                       (match_operand:DI 2 "register_operand" "r")
6961                       ]
6962                       UNSPEC_GOTTINYTLS)))]
6963   ""
6964   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6965   [(set_attr "type" "multiple")
6966    (set_attr "length" "8")]
6967 )
6968
6969 (define_insn "@tlsle12_<mode>"
6970   [(set (match_operand:P 0 "register_operand" "=r")
6971         (unspec:P [(match_operand:P 1 "register_operand" "r")
6972                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6973                    UNSPEC_TLSLE12))]
6974   ""
6975   "add\\t%<w>0, %<w>1, #%L2";
6976   [(set_attr "type" "alu_sreg")
6977    (set_attr "length" "4")]
6978 )
6979
6980 (define_insn "@tlsle24_<mode>"
6981   [(set (match_operand:P 0 "register_operand" "=r")
6982         (unspec:P [(match_operand:P 1 "register_operand" "r")
6983                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6984                    UNSPEC_TLSLE24))]
6985   ""
6986   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6987   [(set_attr "type" "multiple")
6988    (set_attr "length" "8")]
6989 )
6990
6991 (define_insn "@tlsle32_<mode>"
6992   [(set (match_operand:P 0 "register_operand" "=r")
6993         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6994                    UNSPEC_TLSLE32))]
6995   ""
6996   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6997   [(set_attr "type" "multiple")
6998    (set_attr "length" "8")]
6999 )
7000
7001 (define_insn "@tlsle48_<mode>"
7002   [(set (match_operand:P 0 "register_operand" "=r")
7003         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
7004                    UNSPEC_TLSLE48))]
7005   ""
7006   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
7007   [(set_attr "type" "multiple")
7008    (set_attr "length" "12")]
7009 )
7010
7011 (define_expand "@tlsdesc_small_<mode>"
7012   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
7013   "TARGET_TLS_DESC"
7014   {
7015     if (TARGET_SVE)
7016       {
7017         rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode);
7018         rtx_insn *call
7019           = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
7020         RTL_CONST_CALL_P (call) = 1;
7021       }
7022     else
7023       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
7024     DONE;
7025   }
7026 )
7027
7028 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
7029 ;; R0 and LR.
7030 (define_insn "tlsdesc_small_advsimd_<mode>"
7031   [(set (reg:PTR R0_REGNUM)
7032         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
7033                     UNSPEC_TLSDESC))
7034    (clobber (reg:<P_OF_PTR> LR_REGNUM))
7035    (clobber (reg:CC CC_REGNUM))
7036    (clobber (match_scratch:<P_OF_PTR> 1 "=r"))
7037    (use (reg:<P_OF_PTR> FP_REGNUM))]
7038   "TARGET_TLS_DESC && !TARGET_SVE"
7039   {
7040     if (<MODE>mode == CADImode)
7041       return "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\tx0, x0, %L0\;.tlsdesccall\\t%0\;blr\\t%1";
7042     else
7043       return "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1";
7044   }
7045   [(set_attr "type" "call")
7046    (set_attr "length" "16")])
7047
7048 ;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
7049 ;; describing the extra call-preserved guarantees.  This would work
7050 ;; for non-SVE too, but avoiding a call is probably better if we can.
7051 (define_insn "tlsdesc_small_sve_<mode>"
7052   [(set (reg:PTR R0_REGNUM)
7053         (call (mem:DI (unspec:PTR
7054                         [(match_operand 0 "aarch64_valid_symref")]
7055                         UNSPEC_TLSDESC))
7056               (const_int 0)))
7057    (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
7058    (clobber (reg:DI LR_REGNUM))
7059    (clobber (match_scratch:DI 2 "=r"))]
7060   "TARGET_TLS_DESC && TARGET_SVE"
7061   "adrp\\tx0, %A0\;ldr\\t%<w>2, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%2"
7062   [(set_attr "type" "call")
7063    (set_attr "length" "16")])
7064
7065 (define_insn "stack_tie"
7066   [(set (mem:BLK (scratch))
7067         (unspec:BLK [(match_operand 0 "pmode_register_operand" "rk")
7068                      (match_operand 1 "pmode_register_operand" "rk")]
7069                     UNSPEC_PRLG_STK))]
7070   ""
7071   ""
7072   [(set_attr "length" "0")]
7073 )
7074
7075 (define_insn "aarch64_fjcvtzs"
7076   [(set (match_operand:SI 0 "register_operand" "=r")
7077         (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
7078                    UNSPEC_FJCVTZS))
7079    (clobber (reg:CC CC_REGNUM))]
7080   "TARGET_JSCVT"
7081   "fjcvtzs\\t%w0, %d1"
7082   [(set_attr "type" "f_cvtf2i")]
7083 )
7084
7085 ;; Pointer authentication patterns are always provided.  In architecture
7086 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
7087 ;; This lets the user write portable software which authenticates pointers
7088 ;; when run on something which implements ARMv8.3-A, and which runs
7089 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
7090 ;; implemented.
7091
7092 ;; Signing/Authenticating R30 using SP as the salt.
7093
7094 (define_insn "<pauth_mnem_prefix>sp"
7095   [(set (reg:DI R30_REGNUM)
7096         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
7097   ""
7098   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
7099 )
7100
7101 ;; Signing/Authenticating X17 using X16 as the salt.
7102
7103 (define_insn "<pauth_mnem_prefix>1716"
7104   [(set (reg:DI R17_REGNUM)
7105         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
7106   ""
7107   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
7108 )
7109
7110 ;; Stripping the signature in R30.
7111
7112 (define_insn "xpaclri"
7113   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
7114   ""
7115   "hint\t7 // xpaclri"
7116 )
7117
7118 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
7119 ;; all of memory.  This blocks insns from being moved across this point.
7120
7121 (define_insn "blockage"
7122   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
7123   ""
7124   ""
7125   [(set_attr "length" "0")
7126    (set_attr "type" "block")]
7127 )
7128
7129 (define_insn "probe_stack_range_<mode>"
7130   [(set (match_operand:ADDR 0 "register_operand" "=rk")
7131         (unspec_volatile:ADDR [(match_operand:ADDR 1 "register_operand" "0")
7132                                (match_operand:ADDR 2 "register_operand" "r")]
7133                               UNSPECV_PROBE_STACK_RANGE))]
7134   ""
7135 {
7136   return aarch64_output_probe_stack_range (operands[0], operands[2]);
7137 }
7138   [(set_attr "length" "32")]
7139 )
7140
7141 ;; This instruction is used to generate the stack clash stack adjustment and
7142 ;; probing loop.  We can't change the control flow during prologue and epilogue
7143 ;; code generation.  So we must emit a volatile unspec and expand it later on.
7144
7145 ;; MORELLO TODO The pattern here uses Pmode for everything, but the
7146 ;; min_probe_threshold, adjustment, and guard_size should all be in POmode.
7147 (define_insn "@probe_sve_stack_clash_<mode>"
7148   [(set (match_operand:P 0 "register_operand" "=rk")
7149         (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
7150                             (match_operand:P 2 "register_operand" "r")
7151                             (match_operand:P 3 "const_int_operand" "n")
7152                             (match_operand:P 4 "aarch64_plus_immediate" "L")]
7153                              UNSPECV_PROBE_STACK_RANGE))]
7154   "TARGET_SVE"
7155 {
7156   return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
7157                                                operands[3], operands[4]);
7158 }
7159   [(set_attr "length" "28")]
7160 )
7161
7162 ;; Named pattern for expanding thread pointer reference.
7163 (define_expand "get_thread_pointerdi"
7164   [(match_operand:DI 0 "register_operand")]
7165   ""
7166 {
7167   rtx tmp = aarch64_load_tp (operands[0]);
7168   if (tmp != operands[0])
7169     emit_move_insn (operands[0], tmp);
7170   DONE;
7171 })
7172
7173 ;; Named patterns for stack smashing protection.
7174 (define_expand "stack_protect_set"
7175   [(match_operand 0 "memory_operand")
7176    (match_operand 1 "memory_operand")]
7177   ""
7178 {
7179   machine_mode mode = GET_MODE (operands[0]);
7180   if (aarch64_stack_protector_guard != SSP_GLOBAL)
7181   {
7182     /* Generate access through the system register.  */
7183     rtx tmp_reg = gen_reg_rtx (mode);
7184     emit_insn (gen_reg_stack_protect_address (mode, tmp_reg));
7185     emit_insn (gen_add3_insn (tmp_reg, tmp_reg,
7186                               GEN_INT (aarch64_stack_protector_guard_offset)));
7187     operands[1] = gen_rtx_MEM (mode, tmp_reg);
7188   }
7189
7190   emit_insn (gen_stack_protect_set (mode, operands[0], operands[1]));
7191   DONE;
7192 })
7193
7194 (define_insn "@reg_stack_protect_address_<mode>"
7195  [(set (match_operand:PTR 0 "register_operand" "=r")
7196        (unspec:PTR [(const_int 0)]
7197         UNSPEC_SSP_SYSREG))]
7198  "aarch64_stack_protector_guard != SSP_GLOBAL"
7199  {
7200    char buf[150];
7201    snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7202             aarch64_stack_protector_guard_reg_str);
7203    output_asm_insn (buf, operands);
7204    return "";
7205  }
7206  [(set_attr "type" "mrs")])
7207
7208 ;; DO NOT SPLIT THIS PATTERN.  It is important for security reasons that the
7209 ;; canary value does not live beyond the life of this sequence.
7210 (define_insn "@stack_protect_set_<mode>"
7211   [(set (match_operand:PTR 0 "memory_operand" "=m")
7212         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7213          UNSPEC_SP_SET))
7214    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7215   ""
7216   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
7217   [(set_attr "length" "12")
7218    (set_attr "type" "multiple")])
7219
7220 (define_expand "stack_protect_test"
7221   [(match_operand 0 "memory_operand")
7222    (match_operand 1 "memory_operand")
7223    (match_operand 2)]
7224   ""
7225 {
7226   machine_mode mode = GET_MODE (operands[0]);
7227
7228   if (aarch64_stack_protector_guard != SSP_GLOBAL)
7229   {
7230     /* Generate access through the system register. The
7231        sequence we want here is the access
7232        of the stack offset to come with
7233        mrs scratch_reg, <system_register>
7234        add scratch_reg, scratch_reg, :lo12:offset. */
7235     rtx tmp_reg = gen_reg_rtx (mode);
7236
7237     emit_insn (gen_reg_stack_protect_address (mode, tmp_reg));
7238     emit_insn (gen_add3_insn (tmp_reg, tmp_reg,
7239                               GEN_INT (aarch64_stack_protector_guard_offset)));
7240     operands[1] = gen_rtx_MEM (mode, tmp_reg);
7241   }
7242   emit_insn (gen_stack_protect_test (mode, operands[0], operands[1]));
7243
7244   rtx cc_reg = gen_rtx_REG (CCmode, CC_REGNUM);
7245   emit_jump_insn (gen_condjump (gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx),
7246                                 cc_reg, operands[2]));
7247   DONE;
7248 })
7249
7250 ;; DO NOT SPLIT THIS PATTERN.  It is important for security reasons that the
7251 ;; canary value does not live beyond the end of this sequence.
7252 (define_insn "@stack_protect_test_<mode>"
7253   [(set (reg:CC CC_REGNUM)
7254         (unspec:CC [(match_operand:PTR 0 "memory_operand" "m")
7255                     (match_operand:PTR 1 "memory_operand" "m")]
7256                    UNSPEC_SP_TEST))
7257    (clobber (match_scratch:PTR 2 "=&r"))
7258    (clobber (match_scratch:PTR 3 "=&r"))]
7259   ""
7260   "ldr\t%<w>2, %0\;ldr\t%<w>3, %1\;subs\t%<w>2, %<w>2, %<w>3\;mov\t%3, 0"
7261   [(set_attr "length" "16")
7262    (set_attr "type" "multiple")])
7263
7264 ;; Write into the Floating-point Status or Control Register.
7265 (define_insn "@aarch64_set_<fpscr_name><GPI:mode>"
7266   [(unspec_volatile [(match_operand:GPI 0 "register_operand" "r")] SET_FPSCR)]
7267   ""
7268   "msr\\t<fpscr_name>, %0"
7269   [(set_attr "type" "mrs")])
7270
7271 ;; Read into the Floating-point Status or Control Register.
7272 (define_insn "@aarch64_get_<fpscr_name><GPI:mode>"
7273   [(set (match_operand:GPI 0 "register_operand" "=r")
7274         (unspec_volatile:GPI [(const_int 0)] GET_FPSCR))]
7275   ""
7276   "mrs\\t%0, <fpscr_name>"
7277   [(set_attr "type" "mrs")])
7278
7279 ;; Define the subtract-one-and-jump insns so loop.c
7280 ;; knows what to generate.
7281 (define_expand "doloop_end"
7282   [(use (match_operand 0 "" ""))      ; loop pseudo
7283    (use (match_operand 1 "" ""))]     ; label
7284   "optimize > 0 && flag_modulo_sched"
7285 {
7286   rtx s0;
7287   rtx bcomp;
7288   rtx loc_ref;
7289   rtx cc_reg;
7290   rtx insn;
7291   rtx cmp;
7292
7293   /* Currently SMS relies on the do-loop pattern to recognize loops
7294      where (1) the control part consists of all insns defining and/or
7295      using a certain 'count' register and (2) the loop count can be
7296      adjusted by modifying this register prior to the loop.
7297      ??? The possible introduction of a new block to initialize the
7298      new IV can potentially affect branch optimizations.  */
7299
7300   if (GET_MODE (operands[0]) != DImode)
7301     FAIL;
7302
7303   s0 = operands [0];
7304   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7305
7306   cmp = XVECEXP (PATTERN (insn), 0, 0);
7307   cc_reg = SET_DEST (cmp);
7308   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7309   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7310   emit_jump_insn (gen_rtx_SET (pc_rtx,
7311                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7312                                                      loc_ref, pc_rtx)));
7313   DONE;
7314 })
7315
7316 ;; Track speculation through conditional branches.  We assume that
7317 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7318 (define_insn "speculation_tracker"
7319   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7320         (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7321          UNSPEC_SPECULATION_TRACKER))]
7322   ""
7323   {
7324     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7325     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7326     return "";
7327   }
7328   [(set_attr "type" "csel")]
7329 )
7330
7331 ;; Like speculation_tracker, but track the inverse condition.
7332 (define_insn "speculation_tracker_rev"
7333   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7334         (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7335          UNSPEC_SPECULATION_TRACKER_REV))]
7336   ""
7337   {
7338     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7339     output_asm_insn ("csel\\t%1, %1, xzr, %M0", operands);
7340     return "";
7341   }
7342   [(set_attr "type" "csel")]
7343 )
7344
7345 ;; BTI <target> instructions
7346 (define_insn "bti_noarg"
7347   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7348   ""
7349   "hint\t32 // bti"
7350   [(set_attr "type" "no_insn")]
7351 )
7352
7353 (define_insn "bti_c"
7354   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7355   ""
7356   "hint\t34 // bti c"
7357   [(set_attr "type" "no_insn")]
7358 )
7359
7360 (define_insn "bti_j"
7361   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7362   ""
7363   "hint\t36 // bti j"
7364   [(set_attr "type" "no_insn")]
7365 )
7366
7367 (define_insn "bti_jc"
7368   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7369   ""
7370   "hint\t38 // bti jc"
7371   [(set_attr "type" "no_insn")]
7372 )
7373
7374 ;; Hard speculation barrier.
7375 (define_insn "speculation_barrier"
7376   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7377   ""
7378   "isb\;dsb\\tsy"
7379   [(set_attr "length" "8")
7380    (set_attr "type" "block")
7381    (set_attr "speculation_barrier" "true")]
7382 )
7383
7384 ;; Support for __builtin_speculation_safe_value when we have speculation
7385 ;; tracking enabled.  Use the speculation tracker to decide whether to
7386 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7387 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7388   [(set (match_operand:ALLI_TI 0 "register_operand")
7389         (unspec_volatile:ALLI_TI
7390          [(match_operand:ALLI_TI 1 "register_operand")
7391           (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7392           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7393           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7394   ""
7395   "
7396   {
7397     if (operands[2] == const0_rtx)
7398       {
7399         rtx tracker;
7400         if (<MODE>mode == TImode)
7401           tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7402         else
7403           tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7404
7405         emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7406                                                  tracker));
7407         DONE;
7408       }
7409   }
7410   "
7411 )
7412
7413 ;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
7414 ;; encoding for CSDB, but will work in older versions of the assembler.
7415 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7416   [(set (match_operand:ALLI 0 "register_operand" "=r")
7417         (unspec_volatile:ALLI
7418          [(match_operand:ALLI 1 "register_operand" "r")
7419           (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7420           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7421           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7422   ""
7423   {
7424     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7425     output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7426                      operands);
7427     return "";
7428   }
7429   [(set_attr "length" "12")
7430    (set_attr "type" "block")
7431    (set_attr "speculation_barrier" "true")]
7432 )
7433
7434 ;; Pattern to match despeculate_copyti
7435 (define_insn "*despeculate_copyti_insn"
7436   [(set (match_operand:TI 0 "register_operand" "=r")
7437         (unspec_volatile:TI
7438          [(match_operand:TI 1 "register_operand" "r")
7439           (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7440           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7441           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7442   ""
7443   {
7444     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7445     output_asm_insn
7446       ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7447        operands);
7448     return "";
7449   }
7450   [(set_attr "length" "16")
7451    (set_attr "type" "block")
7452    (set_attr "speculation_barrier" "true")]
7453 )
7454
7455 (define_insn "despeculate_simple<ALLI:mode>"
7456   [(set (match_operand:ALLI 0 "register_operand" "=r")
7457         (unspec_volatile:ALLI
7458          [(match_operand:ALLI 1 "register_operand" "r")
7459           (use (match_operand:ALLI 2 "register_operand" ""))]
7460          UNSPECV_SPECULATION_BARRIER))]
7461   ""
7462   "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7463   [(set_attr "type" "block")
7464    (set_attr "length" "8")
7465    (set_attr "speculation_barrier" "true")]
7466 )
7467
7468 (define_insn "despeculate_simpleti"
7469   [(set (match_operand:TI 0 "register_operand" "=r")
7470         (unspec_volatile:TI
7471          [(match_operand:TI 1 "register_operand" "r")
7472           (use (match_operand:DI 2 "register_operand" ""))]
7473          UNSPECV_SPECULATION_BARRIER))]
7474   ""
7475   "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7476   [(set_attr "type" "block")
7477    (set_attr "length" "12")
7478    (set_attr "speculation_barrier" "true")]
7479 )
7480
7481 (define_insn "aarch64_<frintnzs_op><mode>"
7482   [(set (match_operand:VSFDF 0 "register_operand" "=w")
7483         (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7484                       FRINTNZX))]
7485   "TARGET_FRINT && TARGET_FLOAT
7486    && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7487   "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7488   [(set_attr "type" "f_rint<stype>")]
7489 )
7490
7491 ;; Transactional Memory Extension (TME) instructions.
7492
7493 (define_insn "tstart"
7494   [(set (match_operand:DI 0 "register_operand" "=r")
7495         (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7496    (clobber (mem:BLK (scratch)))]
7497   "TARGET_TME"
7498   "tstart\\t%0"
7499   [(set_attr "type" "tme")]
7500 )
7501
7502 (define_insn "ttest"
7503   [(set (match_operand:DI 0 "register_operand" "=r")
7504         (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7505    (clobber (mem:BLK (scratch)))]
7506   "TARGET_TME"
7507   "ttest\\t%0"
7508   [(set_attr "type" "tme")]
7509 )
7510
7511 (define_insn "tcommit"
7512   [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7513    (clobber (mem:BLK (scratch)))]
7514   "TARGET_TME"
7515   "tcommit"
7516   [(set_attr "type" "tme")]
7517 )
7518
7519 (define_insn "tcancel"
7520   [(unspec_volatile:BLK
7521      [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7522    (clobber (mem:BLK (scratch)))]
7523   "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7524   "tcancel\\t#%0"
7525   [(set_attr "type" "tme")]
7526 )
7527
7528 (define_insn "aarch64_rndr"
7529   [(set (match_operand:DI 0 "register_operand" "=r")
7530         (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR))
7531    (set (reg:CC_Z CC_REGNUM)
7532         (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))]
7533   "TARGET_RNG"
7534   "mrs\t%0, RNDR"
7535   [(set_attr "type" "mrs")]
7536 )
7537
7538 (define_insn "aarch64_rndrrs"
7539   [(set (match_operand:DI 0 "register_operand" "=r")
7540         (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS))
7541    (set (reg:CC_Z CC_REGNUM)
7542         (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))]
7543   "TARGET_RNG"
7544   "mrs\t%0, RNDRRS"
7545   [(set_attr "type" "mrs")]
7546 )
7547
7548 ;; Memory Tagging Extension (MTE) instructions.
7549
7550 (define_insn "irg"
7551   [(set (match_operand:DI 0 "register_operand" "=rk")
7552         (ior:DI
7553          (and:DI (match_operand:DI 1 "register_operand" "rk")
7554                  (const_int -1080863910568919041)) ;; 0xf0ff...
7555          (ashift:DI (unspec:QI [(match_operand:DI 2 "register_operand" "r")]
7556                      UNSPEC_GEN_TAG_RND)
7557                     (const_int 56))))]
7558   "TARGET_MEMTAG"
7559   "irg\\t%0, %1, %2"
7560   [(set_attr "type" "memtag")]
7561 )
7562
7563 (define_insn "gmi"
7564   [(set (match_operand:DI 0 "register_operand" "=r")
7565         (ior:DI (ashift:DI
7566                  (const_int 1)
7567                  (and:QI (lshiftrt:DI
7568                           (match_operand:DI 1 "register_operand" "rk")
7569                           (const_int 56)) (const_int 15)))
7570                 (match_operand:DI 2 "register_operand" "r")))]
7571   "TARGET_MEMTAG"
7572   "gmi\\t%0, %1, %2"
7573   [(set_attr "type" "memtag")]
7574 )
7575
7576 (define_insn "addg"
7577   [(set (match_operand:DI 0 "register_operand" "=rk")
7578         (ior:DI
7579          (and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7580                           (match_operand:DI 2 "aarch64_granule16_uimm6" "i"))
7581                  (const_int -1080863910568919041)) ;; 0xf0ff...
7582          (ashift:DI
7583           (unspec:QI
7584            [(and:QI (lshiftrt:DI (match_dup 1) (const_int 56)) (const_int 15))
7585             (match_operand:QI 3 "aarch64_memtag_tag_offset" "i")]
7586            UNSPEC_GEN_TAG)
7587           (const_int 56))))]
7588   "TARGET_MEMTAG"
7589   "addg\\t%0, %1, #%2, #%3"
7590   [(set_attr "type" "memtag")]
7591 )
7592
7593 (define_insn "subp"
7594   [(set (match_operand:DI 0 "register_operand" "=r")
7595         (minus:DI
7596           (and:DI (match_operand:DI 1 "register_operand" "rk")
7597                   (const_int 72057594037927935)) ;; 0x00ff...
7598           (and:DI (match_operand:DI 2 "register_operand" "rk")
7599                   (const_int 72057594037927935))))] ;; 0x00ff...
7600   "TARGET_MEMTAG"
7601   "subp\\t%0, %1, %2"
7602   [(set_attr "type" "memtag")]
7603 )
7604
7605 ;; LDG will use the 16-byte aligned value of the address.
7606 (define_insn "ldg"
7607   [(set (match_operand:DI 0 "register_operand" "+r")
7608         (ior:DI
7609          (and:DI (match_dup 0) (const_int -1080863910568919041)) ;; 0xf0ff...
7610          (ashift:DI
7611           (mem:QI (unspec:DI
7612            [(and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
7613                              (match_operand:DI 2 "aarch64_granule16_simm9" "i"))
7614                     (const_int -16))] UNSPEC_TAG_SPACE))
7615           (const_int 56))))]
7616   "TARGET_MEMTAG"
7617   "ldg\\t%0, [%1, #%2]"
7618   [(set_attr "type" "memtag")]
7619 )
7620
7621 ;; STG doesn't align the address but aborts with alignment fault
7622 ;; when the address is not 16-byte aligned.
7623 (define_insn "stg"
7624   [(set (mem:QI (unspec:DI
7625          [(plus:DI (match_operand:DI 1 "register_operand" "rk")
7626                    (match_operand:DI 2 "aarch64_granule16_simm9" "i"))]
7627          UNSPEC_TAG_SPACE))
7628         (and:QI (lshiftrt:DI (match_operand:DI 0 "register_operand" "rk")
7629                              (const_int 56)) (const_int 15)))]
7630   "TARGET_MEMTAG"
7631   "stg\\t%0, [%1, #%2]"
7632   [(set_attr "type" "memtag")]
7633 )
7634
7635 ;; AdvSIMD Stuff
7636 (include "aarch64-simd.md")
7637
7638 ;; Atomic Operations
7639 (include "atomics.md")
7640
7641 ;; ldp/stp peephole patterns
7642 (include "aarch64-ldpstp.md")
7643
7644 ;; SVE.
7645 (include "aarch64-sve.md")
7646
7647 ;; SVE2.
7648 (include "aarch64-sve2.md")
7649
7650 ;; Morello
7651 (include "aarch64-morello.md")