]> gcc.gnu.org Git - gcc.git/blame - gcc/config/aarch64/aarch64.md
git://gcc.gnu.org / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[AArch64] Add support for the SVE PCS
[gcc.git] / gcc / config / aarch64 / aarch64.md
CommitLineData
43e9d192 1;; Machine description for AArch64 architecture.
a5544970 2;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
43e9d192
IB		 3;; Contributed by ARM Ltd.
4;;
5;; This file is part of GCC.
6;;
7;; GCC is free software; you can redistribute it and/or modify it
8;; under the terms of the GNU General Public License as published by
9;; the Free Software Foundation; either version 3, or (at your option)
10;; any later version.
11;;
12;; GCC is distributed in the hope that it will be useful, but
13;; WITHOUT ANY WARRANTY; without even the implied warranty of
14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15;; General Public License for more details.
16;;
17;; You should have received a copy of the GNU General Public License
18;; along with GCC; see the file COPYING3. If not see
19;; <http://www.gnu.org/licenses/>.
20
21;; Register numbers
22(define_constants
23 [
24 (R0_REGNUM 0)
25 (R1_REGNUM 1)
26 (R2_REGNUM 2)
27 (R3_REGNUM 3)
28 (R4_REGNUM 4)
29 (R5_REGNUM 5)
30 (R6_REGNUM 6)
31 (R7_REGNUM 7)
32 (R8_REGNUM 8)
33 (R9_REGNUM 9)
34 (R10_REGNUM 10)
35 (R11_REGNUM 11)
36 (R12_REGNUM 12)
37 (R13_REGNUM 13)
38 (R14_REGNUM 14)
39 (R15_REGNUM 15)
40 (R16_REGNUM 16)
43e9d192 41 (R17_REGNUM 17)
43e9d192
IB		 42 (R18_REGNUM 18)
43 (R19_REGNUM 19)
44 (R20_REGNUM 20)
45 (R21_REGNUM 21)
46 (R22_REGNUM 22)
47 (R23_REGNUM 23)
48 (R24_REGNUM 24)
49 (R25_REGNUM 25)
50 (R26_REGNUM 26)
51 (R27_REGNUM 27)
52 (R28_REGNUM 28)
53 (R29_REGNUM 29)
54 (R30_REGNUM 30)
43e9d192
IB		 55 (SP_REGNUM 31)
56 (V0_REGNUM 32)
234c9000
AH		 57 (V1_REGNUM 33)
58 (V2_REGNUM 34)
59 (V3_REGNUM 35)
80c13ac5 60 (V4_REGNUM 36)
234c9000
AH		 61 (V5_REGNUM 37)
62 (V6_REGNUM 38)
63 (V7_REGNUM 39)
80c13ac5 64 (V8_REGNUM 40)
234c9000
AH		 65 (V9_REGNUM 41)
66 (V10_REGNUM 42)
67 (V11_REGNUM 43)
80c13ac5 68 (V12_REGNUM 44)
234c9000
AH		 69 (V13_REGNUM 45)
70 (V14_REGNUM 46)
43e9d192 71 (V15_REGNUM 47)
80c13ac5 72 (V16_REGNUM 48)
234c9000
AH		 73 (V17_REGNUM 49)
74 (V18_REGNUM 50)
75 (V19_REGNUM 51)
80c13ac5 76 (V20_REGNUM 52)
234c9000
AH		 77 (V21_REGNUM 53)
78 (V22_REGNUM 54)
79 (V23_REGNUM 55)
80c13ac5 80 (V24_REGNUM 56)
234c9000
AH		 81 (V25_REGNUM 57)
82 (V26_REGNUM 58)
83 (V27_REGNUM 59)
80c13ac5 84 (V28_REGNUM 60)
234c9000
AH		 85 (V29_REGNUM 61)
86 (V30_REGNUM 62)
43e9d192
IB		 87 (V31_REGNUM 63)
88 (SFP_REGNUM 64)
89 (AP_REGNUM 65)
90 (CC_REGNUM 66)
43cacb12
RS		 91 ;; Defined only to make the DWARF description simpler.
92 (VG_REGNUM 67)
93 (P0_REGNUM 68)
80c13ac5
RS		 94 (P1_REGNUM 69)
95 (P2_REGNUM 70)
96 (P3_REGNUM 71)
97 (P4_REGNUM 72)
98 (P5_REGNUM 73)
99 (P6_REGNUM 74)
43cacb12 100 (P7_REGNUM 75)
80c13ac5
RS		 101 (P8_REGNUM 76)
102 (P9_REGNUM 77)
103 (P10_REGNUM 78)
104 (P11_REGNUM 79)
105 (P12_REGNUM 80)
106 (P13_REGNUM 81)
107 (P14_REGNUM 82)
43cacb12 108 (P15_REGNUM 83)
c600df9a 109 (LAST_SAVED_REGNUM 83)
183bfdaf
RS		 110 (FFR_REGNUM 84)
111 ;; "FFR token": a fake register used for representing the scheduling
112 ;; restrictions on FFR-related operations.
113 (FFRT_REGNUM 85)
143d3b15
TC		 114 ;; Scratch register used by stack clash protection to calculate
115 ;; SVE CFA offsets during probing.
116 (STACK_CLASH_SVE_CFA_REGNUM 11)
117 ;; Scratch registers for prologue/epilogue use.
118 (EP0_REGNUM 12)
119 (EP1_REGNUM 13)
3751345d
RE		 120 ;; A couple of call-clobbered registers that we need to reserve when
121 ;; tracking speculation this is not ABI, so is subject to change.
3751345d 122 (SPECULATION_SCRATCH_REGNUM 14)
143d3b15
TC		 123 (SPECULATION_TRACKER_REGNUM 15)
124 ;; Scratch registers used in frame layout.
125 (IP0_REGNUM 16)
126 (IP1_REGNUM 17)
0e510d18 127 (FP_REGNUM 29)
143d3b15 128 (LR_REGNUM 30)
43e9d192
IB		 129 ]
130)
131
132(define_c_enum "unspec" [
8fc16d72
ST		 133 UNSPEC_AUTIA1716
134 UNSPEC_AUTIB1716
135 UNSPEC_AUTIASP
136 UNSPEC_AUTIBSP
08cc4d92 137 UNSPEC_CALLEE_ABI
43e9d192 138 UNSPEC_CASESI
5d357f26
KT		 139 UNSPEC_CRC32B
140 UNSPEC_CRC32CB
141 UNSPEC_CRC32CH
142 UNSPEC_CRC32CW
143 UNSPEC_CRC32CX
144 UNSPEC_CRC32H
145 UNSPEC_CRC32W
146 UNSPEC_CRC32X
3f598afe
JW		 147 UNSPEC_FCVTZS
148 UNSPEC_FCVTZU
e1d5d19e 149 UNSPEC_FJCVTZS
10bd1d96
KT		 150 UNSPEC_FRINT32Z
151 UNSPEC_FRINT32X
152 UNSPEC_FRINT64Z
153 UNSPEC_FRINT64X
58a3bd25 154 UNSPEC_URECPE
0050faf8
JG		 155 UNSPEC_FRECPE
156 UNSPEC_FRECPS
157 UNSPEC_FRECPX
43e9d192
IB		 158 UNSPEC_FRINTA
159 UNSPEC_FRINTI
160 UNSPEC_FRINTM
0659ce6f 161 UNSPEC_FRINTN
43e9d192
IB		 162 UNSPEC_FRINTP
163 UNSPEC_FRINTX
164 UNSPEC_FRINTZ
165 UNSPEC_GOTSMALLPIC
1b1e81f8 166 UNSPEC_GOTSMALLPIC28K
43e9d192 167 UNSPEC_GOTSMALLTLS
87dd8ab0 168 UNSPEC_GOTTINYPIC
5ae7caad 169 UNSPEC_GOTTINYTLS
89b4515c 170 UNSPEC_LD1
43e9d192 171 UNSPEC_LD2
fca7d0a4 172 UNSPEC_LD2_DREG
77efea31 173 UNSPEC_LD2_DUP
43e9d192 174 UNSPEC_LD3
fca7d0a4 175 UNSPEC_LD3_DREG
77efea31 176 UNSPEC_LD3_DUP
43e9d192 177 UNSPEC_LD4
fca7d0a4 178 UNSPEC_LD4_DREG
77efea31 179 UNSPEC_LD4_DUP
3ec1be97
CB		 180 UNSPEC_LD2_LANE
181 UNSPEC_LD3_LANE
182 UNSPEC_LD4_LANE
43e9d192
IB		 183 UNSPEC_MB
184 UNSPEC_NOP
8fc16d72
ST		 185 UNSPEC_PACIA1716
186 UNSPEC_PACIB1716
187 UNSPEC_PACIASP
188 UNSPEC_PACIBSP
43e9d192 189 UNSPEC_PRLG_STK
43cacb12 190 UNSPEC_REV
43e9d192 191 UNSPEC_RBIT
75add2d0
KT		 192 UNSPEC_SABAL
193 UNSPEC_SABDL2
194 UNSPEC_SADALP
3f598afe 195 UNSPEC_SCVTF
2e100703
VP		 196 UNSPEC_SISD_NEG
197 UNSPEC_SISD_SSHL
198 UNSPEC_SISD_USHL
199 UNSPEC_SSHL_2S
89b4515c 200 UNSPEC_ST1
43e9d192
IB		 201 UNSPEC_ST2
202 UNSPEC_ST3
203 UNSPEC_ST4
ba081b77
JG		 204 UNSPEC_ST2_LANE
205 UNSPEC_ST3_LANE
206 UNSPEC_ST4_LANE
43e9d192
IB		 207 UNSPEC_TLS
208 UNSPEC_TLSDESC
cbf5629e
JW		 209 UNSPEC_TLSLE12
210 UNSPEC_TLSLE24
211 UNSPEC_TLSLE32
212 UNSPEC_TLSLE48
75add2d0
KT		 213 UNSPEC_UABAL
214 UNSPEC_UABDL2
215 UNSPEC_UADALP
3f598afe 216 UNSPEC_UCVTF
2e100703 217 UNSPEC_USHL_2S
43e9d192 218 UNSPEC_VSTRUCTDUMMY
cd0b2d36 219 UNSPEC_SSP_SYSREG
36e17020
VK		 220 UNSPEC_SP_SET
221 UNSPEC_SP_TEST
a6fc00da
BH		 222 UNSPEC_RSQRT
223 UNSPEC_RSQRTE
224 UNSPEC_RSQRTS
d9c981c8 225 UNSPEC_NZCV
db58fd89 226 UNSPEC_XPACLRI
43cacb12
RS		 227 UNSPEC_LD1_SVE
228 UNSPEC_ST1_SVE
624d0f07
RS		 229 UNSPEC_LDNT1_SVE
230 UNSPEC_STNT1_SVE
43cacb12 231 UNSPEC_LD1RQ
bfaa08b7 232 UNSPEC_LD1_GATHER
624d0f07 233 UNSPEC_LDFF1_GATHER
f307441a 234 UNSPEC_ST1_SCATTER
06308276 235 UNSPEC_PRED_X
00fa90d9 236 UNSPEC_PRED_Z
34467289 237 UNSPEC_PTEST
624d0f07 238 UNSPEC_PTRUE
43cacb12
RS		 239 UNSPEC_UNPACKSHI
240 UNSPEC_UNPACKUHI
241 UNSPEC_UNPACKSLO
242 UNSPEC_UNPACKULO
243 UNSPEC_PACK
624d0f07 244 UNSPEC_WHILE_LE
43cacb12 245 UNSPEC_WHILE_LO
624d0f07
RS		 246 UNSPEC_WHILE_LS
247 UNSPEC_WHILE_LT
9f4cbab8
RS		 248 UNSPEC_LDN
249 UNSPEC_STN
f1739b48 250 UNSPEC_INSR
624d0f07 251 UNSPEC_CLASTA
bb6c2b68 252 UNSPEC_CLASTB
b781a135 253 UNSPEC_FADDA
002092be 254 UNSPEC_REV_SUBREG
4aeb1ba7 255 UNSPEC_REINTERPRET
3751345d 256 UNSPEC_SPECULATION_TRACKER
315fdae8 257 UNSPEC_COPYSIGN
89626179 258 UNSPEC_TTEST ; Represent transaction test.
624d0f07
RS		 259 UNSPEC_UPDATE_FFR
260 UNSPEC_UPDATE_FFRT
261 UNSPEC_RDFFR
262 UNSPEC_WRFFR
263 ;; Represents an SVE-style lane index, in which the indexing applies
264 ;; within the containing 128-bit block.
265 UNSPEC_SVE_LANE_SELECT
266 UNSPEC_SVE_CNT_PAT
267 UNSPEC_SVE_PREFETCH
268 UNSPEC_SVE_PREFETCH_GATHER
269 UNSPEC_SVE_COMPACT
270 UNSPEC_SVE_SPLICE
43e9d192
IB		 271])
272
273(define_c_enum "unspecv" [
274 UNSPECV_EH_RETURN ; Represent EH_RETURN
aa87aced
KV		 275 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
276 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
277 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
278 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
a3eb8a52 279 UNSPECV_BLOCKAGE ; Represent a blockage
f58101cf 280 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
def745b2 281 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
b5f794b4
SD		 282 UNSPECV_BTI_NOARG ; Represent BTI.
283 UNSPECV_BTI_C ; Represent BTI c.
284 UNSPECV_BTI_J ; Represent BTI j.
285 UNSPECV_BTI_JC ; Represent BTI jc.
89626179
SD		 286 UNSPECV_TSTART ; Represent transaction start.
287 UNSPECV_TCOMMIT ; Represent transaction commit.
288 UNSPECV_TCANCEL ; Represent transaction cancel.
c5dc215d
KT		 289 UNSPEC_RNDR ; Represent RNDR
290 UNSPEC_RNDRRS ; Represent RNDRRS
43e9d192
IB		 291 ]
292)
293
34467289
RS		 294;; These constants are used as a const_int in various SVE unspecs
295;; to indicate whether the governing predicate is known to be a PTRUE.
296(define_constants
297 [; Indicates that the predicate might not be a PTRUE.
298 (SVE_MAYBE_NOT_PTRUE 0)
299
300 ; Indicates that the predicate is known to be a PTRUE.
301 (SVE_KNOWN_PTRUE 1)])
302
c9c5a809
RS		 303;; These constants are used as a const_int in predicated SVE FP arithmetic
304;; to indicate whether the operation is allowed to make additional lanes
305;; active without worrying about the effect on faulting behavior.
306(define_constants
307 [; Indicates either that all lanes are active or that the instruction may
308 ; operate on inactive inputs even if doing so could induce a fault.
309 (SVE_RELAXED_GP 0)
310
311 ; Indicates that some lanes might be inactive and that the instruction
312 ; must not operate on inactive inputs if doing so could induce a fault.
313 (SVE_STRICT_GP 1)])
314
43e9d192
IB		 315;; If further include files are added the defintion of MD_INCLUDES
316;; must be updated.
317
318(include "constraints.md")
319(include "predicates.md")
320(include "iterators.md")
321
43e9d192
IB		 322;; -------------------------------------------------------------------
323;; Instruction types and attributes
324;; -------------------------------------------------------------------
325
026c3cfd 326; The "type" attribute is included here from AArch32 backend to be able
34b512f5
SN		 327; to share pipeline descriptions.
328(include "../arm/types.md")
43e9d192 329
a636bb40
KT		 330;; It is important to set the fp or simd attributes to yes when a pattern
331;; alternative uses the FP or SIMD register files, usually signified by use of
332;; the 'w' constraint. This will ensure that the alternative will be
333;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
334;; architecture extensions. If all the alternatives in a pattern use the
335;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
336;; or TARGET_SIMD.
337
488461d8
MM		 338;; Attributes of the architecture required to support the instruction (or
339;; alternative). This attribute is used to compute attribute "enabled", use type
340;; "any" to enable an alternative in all cases.
43e9d192 341
3c5af608 342(define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
90e6443f 343
488461d8 344(define_enum_attr "arch" "arches" (const_string "any"))
43e9d192 345
488461d8
MM		 346;; [For compatibility with Arm in pipeline models]
347;; Attribute that specifies whether or not the instruction touches fp
348;; registers.
349;; Note that this attribute is not used anywhere in either the arm or aarch64
350;; backends except in the scheduling description for xgene1. In that
351;; scheduling description this attribute is used to subclass the load_4 and
352;; load_8 types.
353(define_attr "fp" "no,yes"
354 (if_then_else
355 (eq_attr "arch" "fp")
356 (const_string "yes")
357 (const_string "no")))
358
359(define_attr "arch_enabled" "no,yes"
360 (if_then_else
361 (ior
362 (eq_attr "arch" "any")
363
3c5af608
MM		 364 (and (eq_attr "arch" "rcpc8_4")
365 (match_test "AARCH64_ISA_RCPC8_4"))
366
488461d8
MM		 367 (and (eq_attr "arch" "fp")
368 (match_test "TARGET_FLOAT"))
369
370 (and (eq_attr "arch" "simd")
371 (match_test "TARGET_SIMD"))
372
373 (and (eq_attr "arch" "fp16")
374 (match_test "TARGET_FP_F16INST"))
375
376 (and (eq_attr "arch" "sve")
377 (match_test "TARGET_SVE")))
378 (const_string "yes")
379 (const_string "no")))
43cacb12 380
43e9d192
IB		 381;; Attribute that controls whether an alternative is enabled or not.
382;; Currently it is only used to disable alternatives which touch fp or simd
488461d8
MM		 383;; registers when -mgeneral-regs-only is specified or to require a special
384;; architecture support.
385(define_attr "enabled" "no,yes" (attr "arch_enabled"))
43e9d192 386
973d2e01
TP		 387;; Attribute that specifies whether we are dealing with a branch to a
388;; label that is far away, i.e. further away than the maximum/minimum
389;; representable in a signed 21-bits number.
390;; 0 :=: no
391;; 1 :=: yes
392(define_attr "far_branch" "" (const_int 0))
393
a08acce8
RH		 394;; Attribute that specifies whether the alternative uses MOVPRFX.
395(define_attr "movprfx" "no,yes" (const_string "no"))
396
397(define_attr "length" ""
398 (cond [(eq_attr "movprfx" "yes")
399 (const_int 8)
400 ] (const_int 4)))
401
4adcec71
EM		 402;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
403;; no predicated insns.
404(define_attr "predicated" "yes,no" (const_string "no"))
405
3751345d
RE		 406;; Set to true on an insn that requires the speculation tracking state to be
407;; in the tracking register before the insn issues. Otherwise the compiler
408;; may chose to hold the tracking state encoded in SP.
409(define_attr "speculation_barrier" "true,false" (const_string "false"))
410
43e9d192
IB		 411;; -------------------------------------------------------------------
412;; Pipeline descriptions and scheduling
413;; -------------------------------------------------------------------
414
415;; Processor types.
416(include "aarch64-tune.md")
417
418;; Scheduling
c451f4d6 419(include "../arm/cortex-a53.md")
e0ae0288 420(include "../arm/cortex-a57.md")
b2ca46df 421(include "../arm/exynos-m1.md")
825ada35 422(include "falkor.md")
2e9d500e 423(include "saphira.md")
2d41ed58 424(include "thunderx.md")
a0f06fc9 425(include "../arm/xgene1.md")
f192030d 426(include "thunderx2t99.md")
8108dfde 427(include "tsv110.md")
43e9d192
IB		 428
429;; -------------------------------------------------------------------
430;; Jumps and other miscellaneous insns
431;; -------------------------------------------------------------------
432
433(define_insn "indirect_jump"
434 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
435 ""
436 "br\\t%0"
c1dfcec7 437 [(set_attr "type" "branch")]
43e9d192
IB		 438)
439
440(define_insn "jump"
441 [(set (pc) (label_ref (match_operand 0 "" "")))]
442 ""
443 "b\\t%l0"
c1dfcec7 444 [(set_attr "type" "branch")]
43e9d192
IB		 445)
446
447(define_expand "cbranch<mode>4"
448 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
1bbffb87
DZ		 449 [(match_operand:GPI 1 "register_operand")
450 (match_operand:GPI 2 "aarch64_plus_operand")])
43e9d192
IB		 451 (label_ref (match_operand 3 "" ""))
452 (pc)))]
453 ""
454 "
455 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
456 operands[2]);
457 operands[2] = const0_rtx;
458 "
459)
460
461(define_expand "cbranch<mode>4"
462 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
1bbffb87
DZ		 463 [(match_operand:GPF 1 "register_operand")
464 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
43e9d192
IB		 465 (label_ref (match_operand 3 "" ""))
466 (pc)))]
467 ""
468 "
469 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
470 operands[2]);
471 operands[2] = const0_rtx;
472 "
473)
474
cf670503
ZC		 475(define_expand "cbranchcc4"
476 [(set (pc) (if_then_else
477 (match_operator 0 "aarch64_comparison_operator"
1bbffb87 478 [(match_operand 1 "cc_register")
cf670503
ZC		 479 (match_operand 2 "const0_operand")])
480 (label_ref (match_operand 3 "" ""))
481 (pc)))]
482 ""
483 "")
484
c8012fbc
WD		 485(define_insn "ccmp<mode>"
486 [(set (match_operand:CC 1 "cc_register" "")
487 (if_then_else:CC
cf670503 488 (match_operator 4 "aarch64_comparison_operator"
c8012fbc 489 [(match_operand 0 "cc_register" "")
cf670503 490 (const_int 0)])
c8012fbc
WD		 491 (compare:CC
492 (match_operand:GPI 2 "register_operand" "r,r,r")
493 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
d9c981c8 494 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
c8012fbc 495 ""
cf670503
ZC		 496 "@
497 ccmp\\t%<w>2, %<w>3, %k5, %m4
e2b691c4 498 ccmp\\t%<w>2, %3, %k5, %m4
cf670503
ZC		 499 ccmn\\t%<w>2, #%n3, %k5, %m4"
500 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
501)
502
786e3c06
WD		 503(define_insn "fccmp<mode>"
504 [(set (match_operand:CCFP 1 "cc_register" "")
505 (if_then_else:CCFP
506 (match_operator 4 "aarch64_comparison_operator"
507 [(match_operand 0 "cc_register" "")
508 (const_int 0)])
509 (compare:CCFP
510 (match_operand:GPF 2 "register_operand" "w")
511 (match_operand:GPF 3 "register_operand" "w"))
d9c981c8 512 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
786e3c06
WD		 513 "TARGET_FLOAT"
514 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
c297d256 515 [(set_attr "type" "fccmp<s>")]
786e3c06
WD		 516)
517
518(define_insn "fccmpe<mode>"
519 [(set (match_operand:CCFPE 1 "cc_register" "")
520 (if_then_else:CCFPE
521 (match_operator 4 "aarch64_comparison_operator"
522 [(match_operand 0 "cc_register" "")
523 (const_int 0)])
524 (compare:CCFPE
525 (match_operand:GPF 2 "register_operand" "w")
526 (match_operand:GPF 3 "register_operand" "w"))
d9c981c8 527 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
786e3c06
WD		 528 "TARGET_FLOAT"
529 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
c297d256 530 [(set_attr "type" "fccmp<s>")]
786e3c06
WD		 531)
532
4f58fe36
KT		 533;; Expansion of signed mod by a power of 2 using CSNEG.
534;; For x0 % n where n is a power of 2 produce:
535;; negs x1, x0
536;; and x0, x0, #(n - 1)
537;; and x1, x1, #(n - 1)
538;; csneg x0, x0, x1, mi
539
540(define_expand "mod<mode>3"
1bbffb87
DZ		 541 [(match_operand:GPI 0 "register_operand")
542 (match_operand:GPI 1 "register_operand")
543 (match_operand:GPI 2 "const_int_operand")]
4f58fe36
KT		 544 ""
545 {
546 HOST_WIDE_INT val = INTVAL (operands[2]);
547
548 if (val <= 0
549 || exact_log2 (val) <= 0
550 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
551 FAIL;
552
553 rtx mask = GEN_INT (val - 1);
554
555 /* In the special case of x0 % 2 we can do the even shorter:
556 cmp x0, xzr
557 and x0, x0, 1
558 cneg x0, x0, lt. */
559 if (val == 2)
560 {
561 rtx masked = gen_reg_rtx (<MODE>mode);
562 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
563 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
564 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
565 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
566 DONE;
567 }
568
569 rtx neg_op = gen_reg_rtx (<MODE>mode);
570 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
571
572 /* Extract the condition register and mode. */
573 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
574 rtx cc_reg = SET_DEST (cmp);
575 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
576
577 rtx masked_pos = gen_reg_rtx (<MODE>mode);
578 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
579
580 rtx masked_neg = gen_reg_rtx (<MODE>mode);
581 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
582
583 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
584 masked_neg, masked_pos));
585 DONE;
586 }
587)
588
714e1b3b 589(define_insn "condjump"
43e9d192
IB		 590 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
591 [(match_operand 1 "cc_register" "") (const_int 0)])
592 (label_ref (match_operand 2 "" ""))
593 (pc)))]
594 ""
973d2e01 595 {
57d6f4d0
RS		 596 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
597 but the "." is required for SVE conditions. */
598 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
973d2e01 599 if (get_attr_length (insn) == 8)
57d6f4d0
RS		 600 return aarch64_gen_far_branch (operands, 2, "Lbcond",
601 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
973d2e01 602 else
57d6f4d0 603 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
973d2e01
TP		 604 }
605 [(set_attr "type" "branch")
606 (set (attr "length")
607 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
608 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
609 (const_int 4)
610 (const_int 8)))
611 (set (attr "far_branch")
612 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
613 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
614 (const_int 0)
615 (const_int 1)))]
43e9d192
IB		 616)
617
714e1b3b
KT		 618;; For a 24-bit immediate CST we can optimize the compare for equality
619;; and branch sequence from:
620;; mov x0, #imm1
621;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
622;; cmp x1, x0
623;; b<ne,eq> .Label
624;; into the shorter:
625;; sub x0, x1, #(CST & 0xfff000)
626;; subs x0, x0, #(CST & 0x000fff)
627;; b<ne,eq> .Label
e7ba492a 628(define_insn_and_split "*compare_condjump<GPI:mode>"
714e1b3b
KT		 629 [(set (pc) (if_then_else (EQL
630 (match_operand:GPI 0 "register_operand" "r")
631 (match_operand:GPI 1 "aarch64_imm24" "n"))
632 (label_ref:P (match_operand 2 "" ""))
633 (pc)))]
e7ba492a
RS		 634 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
635 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
714e1b3b
KT		 636 && !reload_completed"
637 "#"
638 "&& true"
639 [(const_int 0)]
640 {
641 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
642 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
e7ba492a
RS		 643 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
644 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
645 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
714e1b3b 646 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
e7ba492a
RS		 647 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
648 cc_reg, const0_rtx);
714e1b3b
KT		 649 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
650 DONE;
651 }
652)
653
43e9d192 654(define_expand "casesi"
1bbffb87
DZ		 655 [(match_operand:SI 0 "register_operand") ; Index
656 (match_operand:SI 1 "const_int_operand") ; Lower bound
657 (match_operand:SI 2 "const_int_operand") ; Total range
43e9d192
IB		 658 (match_operand:DI 3 "" "") ; Table label
659 (match_operand:DI 4 "" "")] ; Out of range label
660 ""
661 {
662 if (operands[1] != const0_rtx)
663 {
664 rtx reg = gen_reg_rtx (SImode);
665
666 /* Canonical RTL says that if you have:
667
668 (minus (X) (CONST))
669
670 then this should be emitted as:
671
672 (plus (X) (-CONST))
673
674 The use of trunc_int_for_mode ensures that the resulting
675 constant can be represented in SImode, this is important
676 for the corner case where operand[1] is INT_MIN. */
677
678 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
679
680 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
681 (operands[1], SImode))
682 operands[1] = force_reg (SImode, operands[1]);
683 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
684 operands[0] = reg;
685 }
686
687 if (!aarch64_plus_operand (operands[2], SImode))
688 operands[2] = force_reg (SImode, operands[2]);
689 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
690 const0_rtx),
691 operands[0], operands[2], operands[4]));
692
4033e62c 693 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
ec302910
JJ		 694 operands[2]
695 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
696 UNSPEC_CASESI);
697 operands[2] = gen_rtx_MEM (DImode, operands[2]);
698 MEM_READONLY_P (operands[2]) = 1;
699 MEM_NOTRAP_P (operands[2]) = 1;
700 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
43e9d192
IB		 701 DONE;
702 }
703)
704
ec302910
JJ		 705(define_expand "casesi_dispatch"
706 [(parallel
707 [(set (pc) (match_operand:DI 0 ""))
708 (clobber (reg:CC CC_REGNUM))
709 (clobber (match_scratch:DI 2))
710 (clobber (match_scratch:DI 3))
711 (use (label_ref:DI (match_operand 1 "")))])]
712 "")
713
714(define_insn "*casesi_dispatch"
43e9d192
IB		 715 [(parallel
716 [(set (pc)
717 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
718 (match_operand:SI 1 "register_operand" "r")]
719 UNSPEC_CASESI)))
720 (clobber (reg:CC CC_REGNUM))
721 (clobber (match_scratch:DI 3 "=r"))
722 (clobber (match_scratch:DI 4 "=r"))
ec302910 723 (use (label_ref:DI (match_operand 2 "" "")))])]
43e9d192
IB		 724 ""
725 "*
726 return aarch64_output_casesi (operands);
727 "
728 [(set_attr "length" "16")
34b512f5 729 (set_attr "type" "branch")]
43e9d192
IB		 730)
731
732(define_insn "nop"
733 [(unspec[(const_int 0)] UNSPEC_NOP)]
734 ""
735 "nop"
c1dfcec7 736 [(set_attr "type" "no_insn")]
43e9d192
IB		 737)
738
79c6bb5f 739(define_insn "prefetch"
9bf2f779 740 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
79c6bb5f
GG		 741 (match_operand:QI 1 "const_int_operand" "")
742 (match_operand:QI 2 "const_int_operand" ""))]
743 ""
744 {
9bf2f779 745 const char * pftype[2][4] =
79c6bb5f 746 {
9bf2f779
KT		 747 {"prfm\\tPLDL1STRM, %0",
748 "prfm\\tPLDL3KEEP, %0",
749 "prfm\\tPLDL2KEEP, %0",
750 "prfm\\tPLDL1KEEP, %0"},
751 {"prfm\\tPSTL1STRM, %0",
752 "prfm\\tPSTL3KEEP, %0",
753 "prfm\\tPSTL2KEEP, %0",
754 "prfm\\tPSTL1KEEP, %0"},
79c6bb5f
GG		 755 };
756
757 int locality = INTVAL (operands[2]);
758
759 gcc_assert (IN_RANGE (locality, 0, 3));
760
9bf2f779
KT		 761 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
762 the address into a DImode MEM so that aarch64_print_operand knows
763 how to print it. */
764 operands[0] = gen_rtx_MEM (DImode, operands[0]);
79c6bb5f
GG		 765 return pftype[INTVAL(operands[1])][locality];
766 }
89b2133e 767 [(set_attr "type" "load_4")]
79c6bb5f
GG		 768)
769
cd93c60e
AP		 770(define_insn "trap"
771 [(trap_if (const_int 1) (const_int 8))]
772 ""
773 "brk #1000"
774 [(set_attr "type" "trap")])
775
43e9d192
IB		 776(define_expand "prologue"
777 [(clobber (const_int 0))]
778 ""
779 "
780 aarch64_expand_prologue ();
781 DONE;
782 "
783)
784
785(define_expand "epilogue"
786 [(clobber (const_int 0))]
787 ""
788 "
789 aarch64_expand_epilogue (false);
790 DONE;
791 "
792)
793
794(define_expand "sibcall_epilogue"
795 [(clobber (const_int 0))]
796 ""
797 "
798 aarch64_expand_epilogue (true);
799 DONE;
800 "
801)
802
803(define_insn "*do_return"
804 [(return)]
805 ""
db58fd89
JW		 806 {
807 if (aarch64_return_address_signing_enabled ()
808 && TARGET_ARMV8_3
809 && !crtl->calls_eh_return)
8fc16d72
ST		 810 {
811 if (aarch64_ra_sign_key == AARCH64_KEY_B)
812 return "retab";
813 else
814 return "retaa";
815 }
db58fd89
JW		 816 return "ret";
817 }
c1dfcec7 818 [(set_attr "type" "branch")]
43e9d192
IB		 819)
820
4f942779
RL		 821(define_expand "return"
822 [(simple_return)]
823 "aarch64_use_return_insn_p ()"
824 ""
825)
826
827(define_insn "simple_return"
828 [(simple_return)]
ce9d2a37 829 ""
4f942779
RL		 830 "ret"
831 [(set_attr "type" "branch")]
832)
833
43e9d192
IB		 834(define_insn "*cb<optab><mode>1"
835 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
836 (const_int 0))
837 (label_ref (match_operand 1 "" ""))
838 (pc)))]
6e1eaca9 839 "!aarch64_track_speculation"
973d2e01
TP		 840 {
841 if (get_attr_length (insn) == 8)
842 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
843 else
844 return "<cbz>\\t%<w>0, %l1";
845 }
846 [(set_attr "type" "branch")
847 (set (attr "length")
848 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
849 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
850 (const_int 4)
851 (const_int 8)))
852 (set (attr "far_branch")
853 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
854 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
855 (const_int 0)
856 (const_int 1)))]
43e9d192
IB		 857)
858
859(define_insn "*tb<optab><mode>1"
860 [(set (pc) (if_then_else
861 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
862 (const_int 1)
e27bfda2
KT		 863 (match_operand 1
864 "aarch64_simd_shift_imm_<mode>" "n"))
43e9d192
IB		 865 (const_int 0))
866 (label_ref (match_operand 2 "" ""))
867 (pc)))
096e8448 868 (clobber (reg:CC CC_REGNUM))]
6e1eaca9 869 "!aarch64_track_speculation"
096e8448
JW		 870 {
871 if (get_attr_length (insn) == 8)
872 {
973d2e01
TP		 873 if (get_attr_far_branch (insn) == 1)
874 return aarch64_gen_far_branch (operands, 2, "Ltb",
875 "<inv_tb>\\t%<w>0, %1, ");
876 else
877 {
878 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
879 return "tst\t%<w>0, %1\;<bcond>\t%l2";
880 }
096e8448
JW		 881 }
882 else
883 return "<tbz>\t%<w>0, %1, %l2";
884 }
c1dfcec7 885 [(set_attr "type" "branch")
43e9d192
IB		 886 (set (attr "length")
887 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
888 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
889 (const_int 4)
973d2e01
TP		 890 (const_int 8)))
891 (set (attr "far_branch")
892 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
893 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
894 (const_int 0)
895 (const_int 1)))]
896
43e9d192
IB		 897)
898
899(define_insn "*cb<optab><mode>1"
900 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
901 (const_int 0))
902 (label_ref (match_operand 1 "" ""))
903 (pc)))
096e8448 904 (clobber (reg:CC CC_REGNUM))]
6e1eaca9 905 "!aarch64_track_speculation"
096e8448
JW		 906 {
907 if (get_attr_length (insn) == 8)
908 {
973d2e01
TP		 909 if (get_attr_far_branch (insn) == 1)
910 return aarch64_gen_far_branch (operands, 1, "Ltb",
911 "<inv_tb>\\t%<w>0, <sizem1>, ");
912 else
913 {
914 char buf[64];
915 uint64_t val = ((uint64_t) 1)
916 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
917 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
918 output_asm_insn (buf, operands);
919 return "<bcond>\t%l1";
920 }
096e8448
JW		 921 }
922 else
923 return "<tbz>\t%<w>0, <sizem1>, %l1";
924 }
c1dfcec7 925 [(set_attr "type" "branch")
43e9d192
IB		 926 (set (attr "length")
927 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
928 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
929 (const_int 4)
973d2e01
TP		 930 (const_int 8)))
931 (set (attr "far_branch")
932 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
933 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
934 (const_int 0)
935 (const_int 1)))]
43e9d192
IB		 936)
937
938;; -------------------------------------------------------------------
939;; Subroutine calls and sibcalls
940;; -------------------------------------------------------------------
941
942(define_expand "call"
08cc4d92
RS		 943 [(parallel
944 [(call (match_operand 0 "memory_operand")
945 (match_operand 1 "general_operand"))
946 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
947 (clobber (reg:DI LR_REGNUM))])]
43e9d192
IB		 948 ""
949 "
950 {
08cc4d92 951 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
78607708 952 DONE;
43e9d192
IB		 953 }"
954)
955
47210a04
RL		 956(define_insn "*call_insn"
957 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
43e9d192 958 (match_operand 1 "" ""))
08cc4d92 959 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
43e9d192
IB		 960 (clobber (reg:DI LR_REGNUM))]
961 ""
47210a04
RL		 962 "@
963 blr\\t%0
d6591257 964 bl\\t%c0"
47210a04 965 [(set_attr "type" "call, call")]
43e9d192
IB		 966)
967
968(define_expand "call_value"
08cc4d92
RS		 969 [(parallel
970 [(set (match_operand 0 "")
971 (call (match_operand 1 "memory_operand")
972 (match_operand 2 "general_operand")))
973 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
974 (clobber (reg:DI LR_REGNUM))])]
43e9d192
IB		 975 ""
976 "
977 {
08cc4d92 978 aarch64_expand_call (operands[0], operands[1], operands[3], false);
78607708 979 DONE;
43e9d192
IB		 980 }"
981)
982
47210a04 983(define_insn "*call_value_insn"
43e9d192 984 [(set (match_operand 0 "" "")
47210a04 985 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
43e9d192 986 (match_operand 2 "" "")))
08cc4d92 987 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
43e9d192
IB		 988 (clobber (reg:DI LR_REGNUM))]
989 ""
47210a04
RL		 990 "@
991 blr\\t%1
d6591257 992 bl\\t%c1"
47210a04 993 [(set_attr "type" "call, call")]
43e9d192
IB		 994)
995
996(define_expand "sibcall"
08cc4d92
RS		 997 [(parallel
998 [(call (match_operand 0 "memory_operand")
999 (match_operand 1 "general_operand"))
1000 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1001 (return)])]
43e9d192
IB		 1002 ""
1003 {
08cc4d92 1004 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
78607708 1005 DONE;
43e9d192
IB		 1006 }
1007)
1008
1009(define_expand "sibcall_value"
08cc4d92
RS		 1010 [(parallel
1011 [(set (match_operand 0 "")
1012 (call (match_operand 1 "memory_operand")
1013 (match_operand 2 "general_operand")))
1014 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1015 (return)])]
43e9d192
IB		 1016 ""
1017 {
08cc4d92 1018 aarch64_expand_call (operands[0], operands[1], operands[3], true);
78607708 1019 DONE;
43e9d192
IB		 1020 }
1021)
1022
1023(define_insn "*sibcall_insn"
fee9ba42 1024 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
08cc4d92
RS		 1025 (match_operand 1 ""))
1026 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
47210a04 1027 (return)]
fee9ba42
JW		 1028 "SIBLING_CALL_P (insn)"
1029 "@
1030 br\\t%0
d6591257 1031 b\\t%c0"
fee9ba42 1032 [(set_attr "type" "branch, branch")]
43e9d192
IB		 1033)
1034
1035(define_insn "*sibcall_value_insn"
08cc4d92 1036 [(set (match_operand 0 "")
aaea00aa
JG		 1037 (call (mem:DI
1038 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
08cc4d92
RS		 1039 (match_operand 2 "")))
1040 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
47210a04 1041 (return)]
fee9ba42
JW		 1042 "SIBLING_CALL_P (insn)"
1043 "@
1044 br\\t%1
d6591257 1045 b\\t%c1"
fee9ba42 1046 [(set_attr "type" "branch, branch")]
43e9d192
IB		 1047)
1048
1049;; Call subroutine returning any type.
1050
1051(define_expand "untyped_call"
1052 [(parallel [(call (match_operand 0 "")
1053 (const_int 0))
1054 (match_operand 1 "")
1055 (match_operand 2 "")])]
1056 ""
1057{
1058 int i;
1059
08cc4d92
RS		 1060 /* Untyped calls always use the default ABI. It's only possible to use
1061 ABI variants if we know the type of the target function. */
1062 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));
43e9d192
IB		 1063
1064 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1065 {
1066 rtx set = XVECEXP (operands[2], 0, i);
1067 emit_move_insn (SET_DEST (set), SET_SRC (set));
1068 }
1069
1070 /* The optimizer does not know that the call sets the function value
1071 registers we stored in the result block. We avoid problems by
1072 claiming that all hard registers are used and clobbered at this
1073 point. */
1074 emit_insn (gen_blockage ());
1075 DONE;
1076})
1077
1078;; -------------------------------------------------------------------
1079;; Moves
1080;; -------------------------------------------------------------------
1081
1082(define_expand "mov<mode>"
1bbffb87
DZ		 1083 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1084 (match_operand:SHORT 1 "general_operand"))]
43e9d192
IB		 1085 ""
1086 "
1087 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1088 operands[1] = force_reg (<MODE>mode, operands[1]);
43cacb12
RS		 1089
1090 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1091 {
1092 aarch64_expand_mov_immediate (operands[0], operands[1]);
1093 DONE;
1094 }
43e9d192
IB		 1095 "
1096)
1097
1098(define_insn "*mov<mode>_aarch64"
43d0a8ee
WD		 1099 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
1100 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
43e9d192
IB		 1101 "(register_operand (operands[0], <MODE>mode)
1102 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
b7342d25
IB		 1103{
1104 switch (which_alternative)
1105 {
1106 case 0:
1107 return "mov\t%w0, %w1";
1108 case 1:
1109 return "mov\t%w0, %1";
1110 case 2:
1111 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1112 <MODE>mode);
1113 case 3:
43cacb12 1114 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
b7342d25 1115 case 4:
43cacb12 1116 return "ldr<size>\t%w0, %1";
b7342d25 1117 case 5:
43cacb12 1118 return "ldr\t%<size>0, %1";
b7342d25 1119 case 6:
43cacb12 1120 return "str<size>\t%w1, %0";
b7342d25 1121 case 7:
43cacb12 1122 return "str\t%<size>1, %0";
b7342d25 1123 case 8:
43cacb12 1124 return "umov\t%w0, %1.<v>[0]";
b7342d25 1125 case 9:
43cacb12
RS		 1126 return "dup\t%0.<Vallxd>, %w1";
1127 case 10:
21216998 1128 return "dup\t%<Vetype>0, %1.<v>[0]";
b7342d25
IB		 1129 default:
1130 gcc_unreachable ();
1131 }
1132}
43cacb12
RS		 1133 ;; The "mov_imm" type for CNT is just a placeholder.
1134 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1135 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
488461d8 1136 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
43e9d192
IB		 1137)
1138
1139(define_expand "mov<mode>"
1bbffb87
DZ		 1140 [(set (match_operand:GPI 0 "nonimmediate_operand")
1141 (match_operand:GPI 1 "general_operand"))]
43e9d192
IB		 1142 ""
1143 "
01b9402c
KT		 1144 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1145 && CONST_INT_P (operands[1]) && <MODE>mode == DImode
141a3ccf
KT		 1146 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1147 DONE;
1148
43e9d192
IB		 1149 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1150 operands[1] = force_reg (<MODE>mode, operands[1]);
1151
82614948
RR		 1152 /* FIXME: RR we still need to fix up what we are doing with
1153 symbol_refs and other types of constants. */
1154 if (CONSTANT_P (operands[1])
1155 && !CONST_INT_P (operands[1]))
1156 {
1157 aarch64_expand_mov_immediate (operands[0], operands[1]);
1158 DONE;
1159 }
43e9d192
IB		 1160 "
1161)
1162
82614948 1163(define_insn_and_split "*movsi_aarch64"
43cacb12 1164 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
43d0a8ee 1165 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
43e9d192
IB		 1166 "(register_operand (operands[0], SImode)
1167 || aarch64_reg_or_zero (operands[1], SImode))"
1168 "@
28514dda
YZ		 1169 mov\\t%w0, %w1
1170 mov\\t%w0, %w1
43e9d192
IB		 1171 mov\\t%w0, %w1
1172 mov\\t%w0, %1
82614948 1173 #
43cacb12 1174 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
43e9d192 1175 ldr\\t%w0, %1
0b064172 1176 ldr\\t%s0, %1
43e9d192 1177 str\\t%w1, %0
0b064172 1178 str\\t%s1, %0
d6591257 1179 adr\\t%x0, %c1
28514dda 1180 adrp\\t%x0, %A1
43e9d192
IB		 1181 fmov\\t%s0, %w1
1182 fmov\\t%w0, %s1
a2170965
TC		 1183 fmov\\t%s0, %s1
1184 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1185 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
7263fa9f 1186 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
82614948
RR		 1187 [(const_int 0)]
1188 "{
1189 aarch64_expand_mov_immediate (operands[0], operands[1]);
1190 DONE;
1191 }"
43cacb12
RS		 1192 ;; The "mov_imm" type for CNT is just a placeholder.
1193 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1194 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
488461d8 1195 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
43e9d192
IB		 1196)
1197
82614948 1198(define_insn_and_split "*movdi_aarch64"
43cacb12
RS		 1199 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1200 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
43e9d192
IB		 1201 "(register_operand (operands[0], DImode)
1202 || aarch64_reg_or_zero (operands[1], DImode))"
1203 "@
1204 mov\\t%x0, %x1
1205 mov\\t%0, %x1
1206 mov\\t%x0, %1
1207 mov\\t%x0, %1
9de00935 1208 mov\\t%w0, %1
82614948 1209 #
43cacb12 1210 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
43e9d192 1211 ldr\\t%x0, %1
0b064172 1212 ldr\\t%d0, %1
43e9d192 1213 str\\t%x1, %0
0b064172 1214 str\\t%d1, %0
d6591257 1215 adr\\t%x0, %c1
43e9d192
IB		 1216 adrp\\t%x0, %A1
1217 fmov\\t%d0, %x1
1218 fmov\\t%x0, %d1
1219 fmov\\t%d0, %d1
a2170965 1220 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
f2a03a72 1221 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
7263fa9f 1222 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
82614948
RR		 1223 [(const_int 0)]
1224 "{
1225 aarch64_expand_mov_immediate (operands[0], operands[1]);
1226 DONE;
1227 }"
43cacb12
RS		 1228 ;; The "mov_imm" type for CNTD is just a placeholder.
1229 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1230 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1231 neon_move")
488461d8 1232 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
43e9d192
IB		 1233)
1234
1235(define_insn "insv_imm<mode>"
1236 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1237 (const_int 16)
0b81f26c
JG		 1238 (match_operand:GPI 1 "const_int_operand" "n"))
1239 (match_operand:GPI 2 "const_int_operand" "n"))]
50d38551
IB		 1240 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1241 && UINTVAL (operands[1]) % 16 == 0"
a05c0ddf 1242 "movk\\t%<w>0, %X2, lsl %1"
99e6af15 1243 [(set_attr "type" "mov_imm")]
43e9d192
IB		 1244)
1245
1246(define_expand "movti"
1bbffb87
DZ		 1247 [(set (match_operand:TI 0 "nonimmediate_operand")
1248 (match_operand:TI 1 "general_operand"))]
43e9d192
IB		 1249 ""
1250 "
1251 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1252 operands[1] = force_reg (TImode, operands[1]);
43cacb12
RS		 1253
1254 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1255 {
1256 emit_move_insn (gen_lowpart (DImode, operands[0]),
1257 gen_lowpart (DImode, operands[1]));
1258 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1259 DONE;
1260 }
43e9d192
IB		 1261 "
1262)
1263
1264(define_insn "*movti_aarch64"
1265 [(set (match_operand:TI 0
c0bb5bc5 1266 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
43e9d192 1267 (match_operand:TI 1
c0bb5bc5 1268 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
43e9d192
IB		 1269 "(register_operand (operands[0], TImode)
1270 || aarch64_reg_or_zero (operands[1], TImode))"
1271 "@
1272 #
1273 #
1274 #
9fe9816b 1275 mov\\t%0.16b, %1.16b
43e9d192
IB		 1276 ldp\\t%0, %H0, %1
1277 stp\\t%1, %H1, %0
1278 stp\\txzr, xzr, %0
1279 ldr\\t%q0, %1
1280 str\\t%q1, %0"
c1dfcec7 1281 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
db46a2e6
JG		 1282 load_16,store_16,store_16,\
1283 load_16,store_16")
43e9d192 1284 (set_attr "length" "8,8,8,4,4,4,4,4,4")
488461d8 1285 (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
78ec3036 1286)
43e9d192
IB		 1287
1288;; Split a TImode register-register or register-immediate move into
1289;; its component DImode pieces, taking care to handle overlapping
1290;; source and dest registers.
1291(define_split
1292 [(set (match_operand:TI 0 "register_operand" "")
1293 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1294 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1295 [(const_int 0)]
1296{
1297 aarch64_split_128bit_move (operands[0], operands[1]);
1298 DONE;
1299})
1300
1301(define_expand "mov<mode>"
1bbffb87
DZ		 1302 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1303 (match_operand:GPF_TF_F16 1 "general_operand"))]
43e9d192 1304 ""
261fb553 1305 {
43e9d192 1306 if (!TARGET_FLOAT)
261fb553 1307 {
fc29dfc9 1308 aarch64_err_no_fpadvsimd (<MODE>mode);
43e9d192 1309 FAIL;
261fb553 1310 }
43e9d192 1311
490948ca
JW		 1312 if (GET_CODE (operands[0]) == MEM
1313 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1314 && aarch64_float_const_zero_rtx_p (operands[1])))
43e9d192 1315 operands[1] = force_reg (<MODE>mode, operands[1]);
261fb553 1316 }
43e9d192
IB		 1317)
1318
c2ec330c 1319(define_insn "*movhf_aarch64"
d89fe63e
TC		 1320 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1321 (match_operand:HF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
c2ec330c 1322 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
701170a4 1323 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
c2ec330c 1324 "@
523d7207 1325 movi\\t%0.4h, #0
90e6443f 1326 fmov\\t%h0, %w1
d89fe63e 1327 dup\\t%w0.4h, %w1
c2ec330c
AL		 1328 umov\\t%w0, %1.h[0]
1329 mov\\t%0.h[0], %1.h[0]
90e6443f 1330 fmov\\t%h0, %1
980902fb 1331 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
c2ec330c
AL		 1332 ldr\\t%h0, %1
1333 str\\t%h1, %0
1334 ldrh\\t%w0, %1
1335 strh\\t%w1, %0
1336 mov\\t%w0, %w1"
d89fe63e 1337 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
89b2133e 1338 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
488461d8 1339 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
c2ec330c
AL		 1340)
1341
43e9d192 1342(define_insn "*movsf_aarch64"
90e6443f
TC		 1343 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1344 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
43e9d192 1345 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
701170a4 1346 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
43e9d192 1347 "@
523d7207 1348 movi\\t%0.2s, #0
43e9d192
IB		 1349 fmov\\t%s0, %w1
1350 fmov\\t%w0, %s1
1351 fmov\\t%s0, %s1
3520f7cc 1352 fmov\\t%s0, %1
90e6443f 1353 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
43e9d192
IB		 1354 ldr\\t%s0, %1
1355 str\\t%s1, %0
1356 ldr\\t%w0, %1
1357 str\\t%w1, %0
90e6443f
TC		 1358 mov\\t%w0, %w1
1359 mov\\t%w0, %1"
1360 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
89b2133e 1361 f_loads,f_stores,load_4,store_4,mov_reg,\
90e6443f 1362 fconsts")
488461d8 1363 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
43e9d192
IB		 1364)
1365
1366(define_insn "*movdf_aarch64"
90e6443f
TC		 1367 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1368 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
43e9d192 1369 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
701170a4 1370 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
43e9d192 1371 "@
523d7207 1372 movi\\t%d0, #0
43e9d192
IB		 1373 fmov\\t%d0, %x1
1374 fmov\\t%x0, %d1
1375 fmov\\t%d0, %d1
3520f7cc 1376 fmov\\t%d0, %1
90e6443f 1377 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
43e9d192
IB		 1378 ldr\\t%d0, %1
1379 str\\t%d1, %0
1380 ldr\\t%x0, %1
1381 str\\t%x1, %0
90e6443f
TC		 1382 mov\\t%x0, %x1
1383 mov\\t%x0, %1"
1384 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
db46a2e6 1385 f_loadd,f_stored,load_8,store_8,mov_reg,\
90e6443f 1386 fconstd")
488461d8 1387 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
90e6443f
TC		 1388)
1389
1390(define_split
1391 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1392 (match_operand:GPF_HF 1 "general_operand"))]
1393 "can_create_pseudo_p ()
1394 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1395 && !aarch64_float_const_representable_p (operands[1])
1396 && aarch64_float_const_rtx_p (operands[1])"
1397 [(const_int 0)]
1398 {
1399 unsigned HOST_WIDE_INT ival;
1400 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1401 FAIL;
1402
1403 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1404 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1405 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1406 DONE;
1407 }
43e9d192
IB		 1408)
1409
43e9d192
IB		 1410(define_insn "*movtf_aarch64"
1411 [(set (match_operand:TF 0
8734dfac 1412 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
43e9d192 1413 (match_operand:TF 1
8734dfac 1414 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
43e9d192 1415 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
6a0f8c01 1416 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
43e9d192 1417 "@
9fe9816b 1418 mov\\t%0.16b, %1.16b
12dc6974
SN		 1419 #
1420 #
1421 #
43e9d192
IB		 1422 movi\\t%0.2d, #0
1423 fmov\\t%s0, wzr
1424 ldr\\t%q0, %1
1425 str\\t%q1, %0
1426 ldp\\t%0, %H0, %1
6a0f8c01
JW		 1427 stp\\t%1, %H1, %0
1428 stp\\txzr, xzr, %0"
04e34811 1429 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
db46a2e6 1430 f_loadd,f_stored,load_16,store_16,store_16")
6a0f8c01 1431 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
488461d8 1432 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
43e9d192
IB		 1433)
1434
12dc6974
SN		 1435(define_split
1436 [(set (match_operand:TF 0 "register_operand" "")
1437 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1438 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1439 [(const_int 0)]
1440 {
1441 aarch64_split_128bit_move (operands[0], operands[1]);
1442 DONE;
1443 }
1444)
1445
e2c75eea
JG		 1446;; 0 is dst
1447;; 1 is src
76715c32 1448;; 2 is size of copy in bytes
e2c75eea
JG		 1449;; 3 is alignment
1450
76715c32 1451(define_expand "cpymemdi"
e2c75eea
JG		 1452 [(match_operand:BLK 0 "memory_operand")
1453 (match_operand:BLK 1 "memory_operand")
1454 (match_operand:DI 2 "immediate_operand")
1455 (match_operand:DI 3 "immediate_operand")]
1456 "!STRICT_ALIGNMENT"
1457{
76715c32 1458 if (aarch64_expand_cpymem (operands))
e2c75eea
JG		 1459 DONE;
1460 FAIL;
1461}
1462)
1463
43e9d192
IB		 1464;; Operands 1 and 3 are tied together by the final condition; so we allow
1465;; fairly lax checking on the second memory operation.
dfe1da23
JW		 1466(define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1467 [(set (match_operand:SX 0 "register_operand" "=r,w")
1468 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1469 (set (match_operand:SX2 2 "register_operand" "=r,w")
1470 (match_operand:SX2 3 "memory_operand" "m,m"))]
1471 "rtx_equal_p (XEXP (operands[3], 0),
1472 plus_constant (Pmode,
1473 XEXP (operands[1], 0),
1474 GET_MODE_SIZE (<SX:MODE>mode)))"
350013bc
BC		 1475 "@
1476 ldp\\t%w0, %w2, %1
1477 ldp\\t%s0, %s2, %1"
89b2133e 1478 [(set_attr "type" "load_8,neon_load1_2reg")
488461d8 1479 (set_attr "arch" "*,fp")]
43e9d192
IB		 1480)
1481
dfe1da23
JW		 1482;; Storing different modes that can still be merged
1483(define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1484 [(set (match_operand:DX 0 "register_operand" "=r,w")
1485 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1486 (set (match_operand:DX2 2 "register_operand" "=r,w")
1487 (match_operand:DX2 3 "memory_operand" "m,m"))]
1488 "rtx_equal_p (XEXP (operands[3], 0),
1489 plus_constant (Pmode,
1490 XEXP (operands[1], 0),
1491 GET_MODE_SIZE (<DX:MODE>mode)))"
350013bc
BC		 1492 "@
1493 ldp\\t%x0, %x2, %1
1494 ldp\\t%d0, %d2, %1"
db46a2e6 1495 [(set_attr "type" "load_16,neon_load1_2reg")
488461d8 1496 (set_attr "arch" "*,fp")]
350013bc
BC		 1497)
1498
a0d0b980
SE		 1499(define_insn "load_pair_dw_tftf"
1500 [(set (match_operand:TF 0 "register_operand" "=w")
1501 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1502 (set (match_operand:TF 2 "register_operand" "=w")
1503 (match_operand:TF 3 "memory_operand" "m"))]
1504 "TARGET_SIMD
1505 && rtx_equal_p (XEXP (operands[3], 0),
1506 plus_constant (Pmode,
1507 XEXP (operands[1], 0),
1508 GET_MODE_SIZE (TFmode)))"
1509 "ldp\\t%q0, %q2, %1"
1510 [(set_attr "type" "neon_ldp_q")
1511 (set_attr "fp" "yes")]
1512)
1513
43e9d192
IB		 1514;; Operands 0 and 2 are tied together by the final condition; so we allow
1515;; fairly lax checking on the second memory operation.
dfe1da23
JW		 1516(define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1517 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1518 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1519 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1520 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1521 "rtx_equal_p (XEXP (operands[2], 0),
1522 plus_constant (Pmode,
1523 XEXP (operands[0], 0),
1524 GET_MODE_SIZE (<SX:MODE>mode)))"
350013bc
BC		 1525 "@
1526 stp\\t%w1, %w3, %0
1527 stp\\t%s1, %s3, %0"
89b2133e 1528 [(set_attr "type" "store_8,neon_store1_2reg")
488461d8 1529 (set_attr "arch" "*,fp")]
350013bc
BC		 1530)
1531
dfe1da23
JW		 1532;; Storing different modes that can still be merged
1533(define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1534 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1535 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1536 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1537 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1538 "rtx_equal_p (XEXP (operands[2], 0),
1539 plus_constant (Pmode,
1540 XEXP (operands[0], 0),
1541 GET_MODE_SIZE (<DX:MODE>mode)))"
350013bc
BC		 1542 "@
1543 stp\\t%x1, %x3, %0
1544 stp\\t%d1, %d3, %0"
db46a2e6 1545 [(set_attr "type" "store_16,neon_store1_2reg")
488461d8 1546 (set_attr "arch" "*,fp")]
43e9d192
IB		 1547)
1548
a0d0b980
SE		 1549(define_insn "store_pair_dw_tftf"
1550 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1551 (match_operand:TF 1 "register_operand" "w"))
1552 (set (match_operand:TF 2 "memory_operand" "=m")
1553 (match_operand:TF 3 "register_operand" "w"))]
1554 "TARGET_SIMD &&
1555 rtx_equal_p (XEXP (operands[2], 0),
1556 plus_constant (Pmode,
1557 XEXP (operands[0], 0),
1558 GET_MODE_SIZE (TFmode)))"
1559 "stp\\t%q1, %q3, %0"
1560 [(set_attr "type" "neon_stp_q")
1561 (set_attr "fp" "yes")]
1562)
1563
3e322b3f
JW		 1564;; Load pair with post-index writeback. This is primarily used in function
1565;; epilogues.
28514dda 1566(define_insn "loadwb_pair<GPI:mode>_<P:mode>"
43e9d192 1567 [(parallel
28514dda
YZ		 1568 [(set (match_operand:P 0 "register_operand" "=k")
1569 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1570 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
43e9d192 1571 (set (match_operand:GPI 2 "register_operand" "=r")
3e322b3f 1572 (mem:GPI (match_dup 1)))
43e9d192 1573 (set (match_operand:GPI 3 "register_operand" "=r")
28514dda
YZ		 1574 (mem:GPI (plus:P (match_dup 1)
1575 (match_operand:P 5 "const_int_operand" "n"))))])]
3e322b3f 1576 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
e7ba492a
RS		 1577 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1578 [(set_attr "type" "load_<GPI:ldpstp_sz>")]
43e9d192
IB		 1579)
1580
159313d9
JW		 1581(define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1582 [(parallel
1583 [(set (match_operand:P 0 "register_operand" "=k")
1584 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1585 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
159313d9 1586 (set (match_operand:GPF 2 "register_operand" "=w")
3e322b3f 1587 (mem:GPF (match_dup 1)))
159313d9
JW		 1588 (set (match_operand:GPF 3 "register_operand" "=w")
1589 (mem:GPF (plus:P (match_dup 1)
1590 (match_operand:P 5 "const_int_operand" "n"))))])]
3e322b3f 1591 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
e7ba492a 1592 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
159313d9
JW		 1593 [(set_attr "type" "neon_load1_2reg")]
1594)
1595
a0d0b980
SE		 1596(define_insn "loadwb_pair<TX:mode>_<P:mode>"
1597 [(parallel
1598 [(set (match_operand:P 0 "register_operand" "=k")
1599 (plus:P (match_operand:P 1 "register_operand" "0")
1600 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1601 (set (match_operand:TX 2 "register_operand" "=w")
1602 (mem:TX (match_dup 1)))
1603 (set (match_operand:TX 3 "register_operand" "=w")
1604 (mem:TX (plus:P (match_dup 1)
1605 (match_operand:P 5 "const_int_operand" "n"))))])]
1606 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1607 "ldp\\t%q2, %q3, [%1], %4"
1608 [(set_attr "type" "neon_ldp_q")]
1609)
1610
3e322b3f
JW		 1611;; Store pair with pre-index writeback. This is primarily used in function
1612;; prologues.
28514dda 1613(define_insn "storewb_pair<GPI:mode>_<P:mode>"
43e9d192 1614 [(parallel
28514dda
YZ		 1615 [(set (match_operand:P 0 "register_operand" "=&k")
1616 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1617 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
28514dda 1618 (set (mem:GPI (plus:P (match_dup 0)
43e9d192
IB		 1619 (match_dup 4)))
1620 (match_operand:GPI 2 "register_operand" "r"))
28514dda
YZ		 1621 (set (mem:GPI (plus:P (match_dup 0)
1622 (match_operand:P 5 "const_int_operand" "n")))
43e9d192
IB		 1623 (match_operand:GPI 3 "register_operand" "r"))])]
1624 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
e7ba492a
RS		 1625 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1626 [(set_attr "type" "store_<GPI:ldpstp_sz>")]
80c11907
JW		 1627)
1628
1629(define_insn "storewb_pair<GPF:mode>_<P:mode>"
1630 [(parallel
1631 [(set (match_operand:P 0 "register_operand" "=&k")
1632 (plus:P (match_operand:P 1 "register_operand" "0")
44707478 1633 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
80c11907
JW		 1634 (set (mem:GPF (plus:P (match_dup 0)
1635 (match_dup 4)))
1636 (match_operand:GPF 2 "register_operand" "w"))
1637 (set (mem:GPF (plus:P (match_dup 0)
1638 (match_operand:P 5 "const_int_operand" "n")))
1639 (match_operand:GPF 3 "register_operand" "w"))])]
1640 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
e7ba492a 1641 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
80c11907 1642 [(set_attr "type" "neon_store1_2reg<q>")]
43e9d192
IB		 1643)
1644
a0d0b980
SE		 1645(define_insn "storewb_pair<TX:mode>_<P:mode>"
1646 [(parallel
1647 [(set (match_operand:P 0 "register_operand" "=&k")
1648 (plus:P (match_operand:P 1 "register_operand" "0")
1649 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1650 (set (mem:TX (plus:P (match_dup 0)
1651 (match_dup 4)))
1652 (match_operand:TX 2 "register_operand" "w"))
1653 (set (mem:TX (plus:P (match_dup 0)
1654 (match_operand:P 5 "const_int_operand" "n")))
1655 (match_operand:TX 3 "register_operand" "w"))])]
1656 "TARGET_SIMD
1657 && INTVAL (operands[5])
1658 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1659 "stp\\t%q2, %q3, [%0, %4]!"
1660 [(set_attr "type" "neon_stp_q")]
1661)
1662
43e9d192
IB		 1663;; -------------------------------------------------------------------
1664;; Sign/Zero extension
1665;; -------------------------------------------------------------------
1666
1667(define_expand "<optab>sidi2"
1668 [(set (match_operand:DI 0 "register_operand")
1669 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1670 ""
1671)
1672
1673(define_insn "*extendsidi2_aarch64"
1674 [(set (match_operand:DI 0 "register_operand" "=r,r")
1675 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1676 ""
1677 "@
1678 sxtw\t%0, %w1
1679 ldrsw\t%0, %1"
89b2133e 1680 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1681)
1682
350013bc
BC		 1683(define_insn "*load_pair_extendsidi2_aarch64"
1684 [(set (match_operand:DI 0 "register_operand" "=r")
1685 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1686 (set (match_operand:DI 2 "register_operand" "=r")
1687 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1688 "rtx_equal_p (XEXP (operands[3], 0),
1689 plus_constant (Pmode,
1690 XEXP (operands[1], 0),
1691 GET_MODE_SIZE (SImode)))"
1692 "ldpsw\\t%0, %2, %1"
89b2133e 1693 [(set_attr "type" "load_8")]
350013bc
BC		 1694)
1695
43e9d192 1696(define_insn "*zero_extendsidi2_aarch64"
0cfc095c
WD		 1697 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1698 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
43e9d192
IB		 1699 ""
1700 "@
1701 uxtw\t%0, %w1
0cfc095c
WD		 1702 ldr\t%w0, %1
1703 fmov\t%s0, %w1
1704 ldr\t%s0, %1
1705 fmov\t%w0, %s1
1706 fmov\t%s0, %s1"
e6b85350 1707 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
0cfc095c 1708 (set_attr "arch" "*,*,fp,fp,fp,fp")]
43e9d192
IB		 1709)
1710
350013bc 1711(define_insn "*load_pair_zero_extendsidi2_aarch64"
0cfc095c
WD		 1712 [(set (match_operand:DI 0 "register_operand" "=r,w")
1713 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1714 (set (match_operand:DI 2 "register_operand" "=r,w")
1715 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
350013bc
BC		 1716 "rtx_equal_p (XEXP (operands[3], 0),
1717 plus_constant (Pmode,
1718 XEXP (operands[1], 0),
1719 GET_MODE_SIZE (SImode)))"
0cfc095c
WD		 1720 "@
1721 ldp\t%w0, %w2, %1
1722 ldp\t%s0, %s2, %1"
1723 [(set_attr "type" "load_8,neon_load1_2reg")
1724 (set_attr "arch" "*,fp")]
350013bc
BC		 1725)
1726
43e9d192
IB		 1727(define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1728 [(set (match_operand:GPI 0 "register_operand")
1729 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1730 ""
1731)
1732
1733(define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1734 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1735 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1736 ""
1737 "@
1738 sxt<SHORT:size>\t%<GPI:w>0, %w1
1739 ldrs<SHORT:size>\t%<GPI:w>0, %1"
89b2133e 1740 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1741)
1742
1743(define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
43d0a8ee 1744 [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
c902d3c8 1745 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
43e9d192
IB		 1746 ""
1747 "@
63715e5e 1748 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
c902d3c8
SN		 1749 ldr<SHORT:size>\t%w0, %1
1750 ldr\t%<SHORT:size>0, %1"
0cfc095c
WD		 1751 [(set_attr "type" "logic_imm,load_4,f_loads")
1752 (set_attr "arch" "*,*,fp")]
43e9d192
IB		 1753)
1754
1755(define_expand "<optab>qihi2"
1756 [(set (match_operand:HI 0 "register_operand")
1757 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1758 ""
1759)
1760
63715e5e 1761(define_insn "*extendqihi2_aarch64"
43e9d192 1762 [(set (match_operand:HI 0 "register_operand" "=r,r")
63715e5e 1763 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
43e9d192
IB		 1764 ""
1765 "@
63715e5e
WD		 1766 sxtb\t%w0, %w1
1767 ldrsb\t%w0, %1"
89b2133e 1768 [(set_attr "type" "extend,load_4")]
43e9d192
IB		 1769)
1770
63715e5e
WD		 1771(define_insn "*zero_extendqihi2_aarch64"
1772 [(set (match_operand:HI 0 "register_operand" "=r,r")
1773 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1774 ""
1775 "@
1776 and\t%w0, %w1, 255
1777 ldrb\t%w0, %1"
89b2133e 1778 [(set_attr "type" "logic_imm,load_4")]
63715e5e
WD		 1779)
1780
43e9d192
IB		 1781;; -------------------------------------------------------------------
1782;; Simple arithmetic
1783;; -------------------------------------------------------------------
1784
1785(define_expand "add<mode>3"
1786 [(set
1bbffb87
DZ		 1787 (match_operand:GPI 0 "register_operand")
1788 (plus:GPI (match_operand:GPI 1 "register_operand")
1789 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
43e9d192 1790 ""
89f0a62e 1791{
35323bd8
KT		 1792 /* If operands[1] is a subreg extract the inner RTX. */
1793 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1794
45a6c1e3
WD		 1795 /* If the constant is too large for a single instruction and isn't frame
1796 based, split off the immediate so it is available for CSE. */
1797 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
0fdc30bc 1798 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
45a6c1e3 1799 && can_create_pseudo_p ()
35323bd8
KT		 1800 && (!REG_P (op1)
1801 || !REGNO_PTR_FRAME_P (REGNO (op1))))
45a6c1e3 1802 operands[2] = force_reg (<MODE>mode, operands[2]);
43cacb12
RS		 1803 /* Expand polynomial additions now if the destination is the stack
1804 pointer, since we don't want to use that as a temporary. */
1805 else if (operands[0] == stack_pointer_rtx
1806 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1807 {
1808 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1809 operands[2], NULL_RTX, NULL_RTX);
1810 DONE;
1811 }
89f0a62e 1812})
43e9d192 1813
89f0a62e 1814(define_insn "*add<mode>3_aarch64"
43e9d192 1815 [(set
0fdc30bc 1816 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
89f0a62e 1817 (plus:GPI
0fdc30bc
RS		 1818 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1819 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
43e9d192
IB		 1820 ""
1821 "@
89f0a62e
RH		 1822 add\\t%<w>0, %<w>1, %2
1823 add\\t%<w>0, %<w>1, %<w>2
1824 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1825 sub\\t%<w>0, %<w>1, #%n2
43cacb12 1826 #
0fdc30bc
RS		 1827 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1828 * return aarch64_output_sve_addvl_addpl (operands[2]);"
1829 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1830 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1831 (set_attr "arch" "*,*,simd,*,*,sve,sve")]
43e9d192
IB		 1832)
1833
3b585938
IB		 1834;; zero_extend version of above
1835(define_insn "*addsi3_aarch64_uxtw"
1836 [(set
89f0a62e 1837 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
3b585938 1838 (zero_extend:DI
89f0a62e 1839 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
ff1335e2 1840 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
3b585938
IB		 1841 ""
1842 "@
1843 add\\t%w0, %w1, %2
1844 add\\t%w0, %w1, %w2
89f0a62e
RH		 1845 sub\\t%w0, %w1, #%n2
1846 #"
1847 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1848)
1849
1850;; If there's a free register, and we can load the constant with a
1851;; single instruction, do so. This has a chance to improve scheduling.
1852(define_peephole2
1853 [(match_scratch:GPI 3 "r")
1854 (set (match_operand:GPI 0 "register_operand")
1855 (plus:GPI
1856 (match_operand:GPI 1 "register_operand")
1857 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1858 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1859 [(set (match_dup 3) (match_dup 2))
1860 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1861)
1862
1863(define_peephole2
1864 [(match_scratch:SI 3 "r")
1865 (set (match_operand:DI 0 "register_operand")
1866 (zero_extend:DI
1867 (plus:SI
1868 (match_operand:SI 1 "register_operand")
1869 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1870 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1871 [(set (match_dup 3) (match_dup 2))
1872 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1873)
1874
1875;; After peephole2 has had a chance to run, split any remaining long
1876;; additions into two add immediates.
1877(define_split
1878 [(set (match_operand:GPI 0 "register_operand")
1879 (plus:GPI
1880 (match_operand:GPI 1 "register_operand")
1881 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1882 "epilogue_completed"
1883 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1884 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1885 {
1886 HOST_WIDE_INT i = INTVAL (operands[2]);
1887 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1888 operands[3] = GEN_INT (i - s);
1889 operands[4] = GEN_INT (s);
1890 }
3b585938
IB		 1891)
1892
43cacb12
RS		 1893;; Match addition of polynomial offsets that require one temporary, for which
1894;; we can use the early-clobbered destination register. This is a separate
1895;; pattern so that the early clobber doesn't affect register allocation
1896;; for other forms of addition. However, we still need to provide an
1897;; all-register alternative, in case the offset goes out of range after
1898;; elimination. For completeness we might as well provide all GPR-based
1899;; alternatives from the main pattern.
1900;;
1901;; We don't have a pattern for additions requiring two temporaries since at
1902;; present LRA doesn't allow new scratches to be added during elimination.
1903;; Such offsets should be rare anyway.
1904;;
1905;; ??? But if we added LRA support for new scratches, much of the ugliness
1906;; here would go away. We could just handle all polynomial constants in
1907;; this pattern.
1908(define_insn_and_split "*add<mode>3_poly_1"
1909 [(set
0fdc30bc 1910 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
43cacb12 1911 (plus:GPI
0fdc30bc
RS		 1912 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
1913 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
43cacb12
RS		 1914 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1915 "@
1916 add\\t%<w>0, %<w>1, %2
1917 add\\t%<w>0, %<w>1, %<w>2
1918 sub\\t%<w>0, %<w>1, #%n2
1919 #
0fdc30bc
RS		 1920 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1921 * return aarch64_output_sve_addvl_addpl (operands[2]);
43cacb12
RS		 1922 #"
1923 "&& epilogue_completed
1924 && !reg_overlap_mentioned_p (operands[0], operands[1])
1925 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1926 [(const_int 0)]
1927 {
1928 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1929 operands[2], operands[0], NULL_RTX);
1930 DONE;
1931 }
0fdc30bc
RS		 1932 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1933 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
43cacb12
RS		 1934)
1935
89f0a62e
RH		 1936(define_split
1937 [(set (match_operand:DI 0 "register_operand")
1938 (zero_extend:DI
1939 (plus:SI
1940 (match_operand:SI 1 "register_operand")
1941 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1942 "epilogue_completed"
1943 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1944 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1945 {
1946 HOST_WIDE_INT i = INTVAL (operands[2]);
1947 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1948 operands[3] = GEN_INT (i - s);
1949 operands[4] = GEN_INT (s);
1950 operands[5] = gen_lowpart (SImode, operands[0]);
1951 }
43e9d192
IB		 1952)
1953
30c46053
MC		 1954(define_expand "addv<mode>4"
1955 [(match_operand:GPI 0 "register_operand")
1956 (match_operand:GPI 1 "register_operand")
a58fe3c5 1957 (match_operand:GPI 2 "aarch64_plus_operand")
30c46053
MC		 1958 (label_ref (match_operand 3 "" ""))]
1959 ""
1960{
a58fe3c5
RE		 1961 if (CONST_INT_P (operands[2]))
1962 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1963 operands[2]));
1964 else
1965 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
30c46053
MC		 1966 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1967
1968 DONE;
1969})
1970
1971(define_expand "uaddv<mode>4"
1972 [(match_operand:GPI 0 "register_operand")
1973 (match_operand:GPI 1 "register_operand")
1974 (match_operand:GPI 2 "register_operand")
1975 (label_ref (match_operand 3 "" ""))]
1976 ""
1977{
1978 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
f7343f20 1979 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
30c46053
MC		 1980
1981 DONE;
1982})
1983
2195867f 1984(define_expand "addti3"
1bbffb87
DZ		 1985 [(set (match_operand:TI 0 "register_operand")
1986 (plus:TI (match_operand:TI 1 "register_operand")
1987 (match_operand:TI 2 "aarch64_reg_or_imm")))]
30c46053
MC		 1988 ""
1989{
1990 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1991
1992 aarch64_addti_scratch_regs (operands[1], operands[2],
1993 &low_dest, &op1_low, &op2_low,
1994 &high_dest, &op1_high, &op2_high);
1995
1996 if (op2_low == const0_rtx)
1997 {
1998 low_dest = op1_low;
1999 if (!aarch64_pluslong_operand (op2_high, DImode))
2000 op2_high = force_reg (DImode, op2_high);
2001 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
2002 }
2003 else
2004 {
2005 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2006 force_reg (DImode, op2_low)));
2007 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
2008 force_reg (DImode, op2_high)));
2009 }
2010
2011 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2012 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2013
2014 DONE;
2015})
2016
2017(define_expand "addvti4"
1bbffb87
DZ		 2018 [(match_operand:TI 0 "register_operand")
2019 (match_operand:TI 1 "register_operand")
2020 (match_operand:TI 2 "aarch64_reg_or_imm")
30c46053 2021 (label_ref (match_operand 3 "" ""))]
2195867f
RH		 2022 ""
2023{
30c46053 2024 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2195867f 2025
30c46053
MC		 2026 aarch64_addti_scratch_regs (operands[1], operands[2],
2027 &low_dest, &op1_low, &op2_low,
2028 &high_dest, &op1_high, &op2_high);
2195867f 2029
30c46053
MC		 2030 if (op2_low == const0_rtx)
2031 {
2032 low_dest = op1_low;
2033 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2034 force_reg (DImode, op2_high)));
2035 }
2036 else
2037 {
2038 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2039 force_reg (DImode, op2_low)));
2040 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2041 force_reg (DImode, op2_high)));
2042 }
2043
2044 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2045 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2046
2047 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2195867f
RH		 2048 DONE;
2049})
2050
30c46053 2051(define_expand "uaddvti4"
1bbffb87
DZ		 2052 [(match_operand:TI 0 "register_operand")
2053 (match_operand:TI 1 "register_operand")
2054 (match_operand:TI 2 "aarch64_reg_or_imm")
30c46053
MC		 2055 (label_ref (match_operand 3 "" ""))]
2056 ""
2057{
2058 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2059
2060 aarch64_addti_scratch_regs (operands[1], operands[2],
2061 &low_dest, &op1_low, &op2_low,
2062 &high_dest, &op1_high, &op2_high);
2063
2064 if (op2_low == const0_rtx)
2065 {
2066 low_dest = op1_low;
2067 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2068 force_reg (DImode, op2_high)));
2069 }
2070 else
2071 {
2072 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2073 force_reg (DImode, op2_low)));
2074 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2075 force_reg (DImode, op2_high)));
2076 }
2077
2078 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2079 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2080
f7343f20 2081 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
30c46053
MC		 2082 DONE;
2083 })
2084
2195867f 2085(define_insn "add<mode>3_compare0"
43e9d192
IB		 2086 [(set (reg:CC_NZ CC_REGNUM)
2087 (compare:CC_NZ
341fca1b 2088 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
34b512f5 2089 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
43e9d192 2090 (const_int 0)))
34b512f5 2091 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
43e9d192
IB		 2092 (plus:GPI (match_dup 1) (match_dup 2)))]
2093 ""
2094 "@
2095 adds\\t%<w>0, %<w>1, %<w>2
e2b691c4 2096 adds\\t%<w>0, %<w>1, %2
43e9d192 2097 subs\\t%<w>0, %<w>1, #%n2"
1d61feeb 2098 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 2099)
2100
3b585938
IB		 2101;; zero_extend version of above
2102(define_insn "*addsi3_compare0_uxtw"
2103 [(set (reg:CC_NZ CC_REGNUM)
2104 (compare:CC_NZ
341fca1b 2105 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
34b512f5 2106 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
3b585938 2107 (const_int 0)))
34b512f5 2108 (set (match_operand:DI 0 "register_operand" "=r,r,r")
3b585938
IB		 2109 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2110 ""
2111 "@
2112 adds\\t%w0, %w1, %w2
e2b691c4 2113 adds\\t%w0, %w1, %2
3b585938 2114 subs\\t%w0, %w1, #%n2"
1d61feeb 2115 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3b585938
IB		 2116)
2117
ef22810a
RH		 2118(define_insn "*add<mode>3_compareC_cconly"
2119 [(set (reg:CC_C CC_REGNUM)
f7343f20
RE		 2120 (compare:CC_C
2121 (plus:GPI
2122 (match_operand:GPI 0 "register_operand" "r,r,r")
2123 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2124 (match_dup 0)))]
ef22810a 2125 ""
ef22810a 2126 "@
f7343f20
RE		 2127 cmn\\t%<w>0, %<w>1
2128 cmn\\t%<w>0, %1
2129 cmp\\t%<w>0, #%n1"
2130 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
ef22810a 2131)
954224d7 2132
ef22810a
RH		 2133(define_insn "add<mode>3_compareC"
2134 [(set (reg:CC_C CC_REGNUM)
30c46053 2135 (compare:CC_C
f7343f20 2136 (plus:GPI
341fca1b 2137 (match_operand:GPI 1 "register_operand" "rk,rk,rk")
f7343f20
RE		 2138 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2139 (match_dup 1)))
2140 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
ef22810a
RH		 2141 (plus:GPI (match_dup 1) (match_dup 2)))]
2142 ""
f7343f20
RE		 2143 "@
2144 adds\\t%<w>0, %<w>1, %<w>2
2145 adds\\t%<w>0, %<w>1, %2
2146 subs\\t%<w>0, %<w>1, #%n2"
2147 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
ef22810a
RH		 2148)
2149
30c46053
MC		 2150(define_insn "*add<mode>3_compareV_cconly_imm"
2151 [(set (reg:CC_V CC_REGNUM)
2152 (compare:CC_V
2153 (plus:<DWI>
2154 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2155 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2156 (sign_extend:<DWI>
2157 (plus:GPI
2158 (match_dup 0)
2159 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2160 "INTVAL (operands[1]) == INTVAL (operands[2])"
2161 "@
2162 cmn\\t%<w>0, %<w>1
2163 cmp\\t%<w>0, #%n1"
2164 [(set_attr "type" "alus_imm")]
2165)
2166
2167(define_insn "*add<mode>3_compareV_cconly"
2168 [(set (reg:CC_V CC_REGNUM)
2169 (compare:CC_V
2170 (plus:<DWI>
2171 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2172 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2173 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2174 ""
2175 "cmn\\t%<w>0, %<w>1"
2176 [(set_attr "type" "alus_sreg")]
2177)
2178
a58fe3c5 2179(define_insn "add<mode>3_compareV_imm"
30c46053
MC		 2180 [(set (reg:CC_V CC_REGNUM)
2181 (compare:CC_V
2182 (plus:<DWI>
2183 (sign_extend:<DWI>
341fca1b 2184 (match_operand:GPI 1 "register_operand" "rk,rk"))
30c46053
MC		 2185 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2186 (sign_extend:<DWI>
2187 (plus:GPI (match_dup 1) (match_dup 2)))))
2188 (set (match_operand:GPI 0 "register_operand" "=r,r")
2189 (plus:GPI (match_dup 1) (match_dup 2)))]
2190 ""
2191 "@
2192 adds\\t%<w>0, %<w>1, %<w>2
2193 subs\\t%<w>0, %<w>1, #%n2"
2194 [(set_attr "type" "alus_imm,alus_imm")]
2195)
2196
2197(define_insn "add<mode>3_compareV"
2198 [(set (reg:CC_V CC_REGNUM)
2199 (compare:CC_V
2200 (plus:<DWI>
341fca1b 2201 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
30c46053
MC		 2202 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2203 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2204 (set (match_operand:GPI 0 "register_operand" "=r")
2205 (plus:GPI (match_dup 1) (match_dup 2)))]
2206 ""
2207 "adds\\t%<w>0, %<w>1, %<w>2"
2208 [(set_attr "type" "alus_sreg")]
2209)
2210
dd1bdce4
VK		 2211(define_insn "*adds_shift_imm_<mode>"
2212 [(set (reg:CC_NZ CC_REGNUM)
2213 (compare:CC_NZ
30c46053 2214 (plus:GPI (ASHIFT:GPI
dd1bdce4
VK		 2215 (match_operand:GPI 1 "register_operand" "r")
2216 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2217 (match_operand:GPI 3 "register_operand" "r"))
2218 (const_int 0)))
2219 (set (match_operand:GPI 0 "register_operand" "=r")
2220 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2221 (match_dup 3)))]
2222 ""
2223 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2224 [(set_attr "type" "alus_shift_imm")]
2225)
2226
2227(define_insn "*subs_shift_imm_<mode>"
2228 [(set (reg:CC_NZ CC_REGNUM)
2229 (compare:CC_NZ
2230 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2231 (ASHIFT:GPI
2232 (match_operand:GPI 2 "register_operand" "r")
2233 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2234 (const_int 0)))
2235 (set (match_operand:GPI 0 "register_operand" "=r")
2236 (minus:GPI (match_dup 1)
2237 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2238 ""
2239 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2240 [(set_attr "type" "alus_shift_imm")]
2241)
2242
860ad33f
N		 2243(define_insn "*adds_mul_imm_<mode>"
2244 [(set (reg:CC_NZ CC_REGNUM)
2245 (compare:CC_NZ
2246 (plus:GPI (mult:GPI
2247 (match_operand:GPI 1 "register_operand" "r")
2248 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
24c56925 2249 (match_operand:GPI 3 "register_operand" "r"))
860ad33f
N		 2250 (const_int 0)))
2251 (set (match_operand:GPI 0 "register_operand" "=r")
2252 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2253 (match_dup 3)))]
2254 ""
2255 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
99e6af15 2256 [(set_attr "type" "alus_shift_imm")]
860ad33f
N		 2257)
2258
2259(define_insn "*subs_mul_imm_<mode>"
2260 [(set (reg:CC_NZ CC_REGNUM)
2261 (compare:CC_NZ
24c56925 2262 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
860ad33f
N		 2263 (mult:GPI
2264 (match_operand:GPI 2 "register_operand" "r")
2265 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2266 (const_int 0)))
2267 (set (match_operand:GPI 0 "register_operand" "=r")
2268 (minus:GPI (match_dup 1)
2269 (mult:GPI (match_dup 2) (match_dup 3))))]
2270 ""
2271 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
99e6af15 2272 [(set_attr "type" "alus_shift_imm")]
860ad33f
N		 2273)
2274
f3d096b0
N		 2275(define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2276 [(set (reg:CC_NZ CC_REGNUM)
2277 (compare:CC_NZ
2278 (plus:GPI
2279 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
341fca1b 2280 (match_operand:GPI 2 "register_operand" "rk"))
f3d096b0
N		 2281 (const_int 0)))
2282 (set (match_operand:GPI 0 "register_operand" "=r")
2283 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2284 ""
2285 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
99e6af15 2286 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2287)
2288
2289(define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2290 [(set (reg:CC_NZ CC_REGNUM)
2291 (compare:CC_NZ
341fca1b 2292 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
f3d096b0
N		 2293 (ANY_EXTEND:GPI
2294 (match_operand:ALLX 2 "register_operand" "r")))
2295 (const_int 0)))
2296 (set (match_operand:GPI 0 "register_operand" "=r")
2297 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2298 ""
2299 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
99e6af15 2300 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2301)
2302
dd1bdce4
VK		 2303(define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2304 [(set (reg:CC_NZ CC_REGNUM)
2305 (compare:CC_NZ
2306 (plus:GPI (ashift:GPI
2307 (ANY_EXTEND:GPI
2308 (match_operand:ALLX 1 "register_operand" "r"))
2309 (match_operand 2 "aarch64_imm3" "Ui3"))
341fca1b 2310 (match_operand:GPI 3 "register_operand" "rk"))
dd1bdce4
VK		 2311 (const_int 0)))
2312 (set (match_operand:GPI 0 "register_operand" "=rk")
2313 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2314 (match_dup 2))
2315 (match_dup 3)))]
2316 ""
2317 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2318 [(set_attr "type" "alus_ext")]
2319)
2320
2321(define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2322 [(set (reg:CC_NZ CC_REGNUM)
2323 (compare:CC_NZ
341fca1b 2324 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
dd1bdce4
VK		 2325 (ashift:GPI
2326 (ANY_EXTEND:GPI
2327 (match_operand:ALLX 2 "register_operand" "r"))
2328 (match_operand 3 "aarch64_imm3" "Ui3")))
2329 (const_int 0)))
2330 (set (match_operand:GPI 0 "register_operand" "=rk")
2331 (minus:GPI (match_dup 1)
2332 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2333 (match_dup 3))))]
2334 ""
2335 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2336 [(set_attr "type" "alus_ext")]
2337)
2338
f3d096b0
N		 2339(define_insn "*adds_<optab><mode>_multp2"
2340 [(set (reg:CC_NZ CC_REGNUM)
2341 (compare:CC_NZ
2342 (plus:GPI (ANY_EXTRACT:GPI
2343 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2344 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2345 (match_operand 3 "const_int_operand" "n")
2346 (const_int 0))
341fca1b 2347 (match_operand:GPI 4 "register_operand" "rk"))
f3d096b0
N		 2348 (const_int 0)))
2349 (set (match_operand:GPI 0 "register_operand" "=r")
2350 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2351 (match_dup 3)
2352 (const_int 0))
2353 (match_dup 4)))]
2354 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2355 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
99e6af15 2356 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2357)
2358
2359(define_insn "*subs_<optab><mode>_multp2"
2360 [(set (reg:CC_NZ CC_REGNUM)
2361 (compare:CC_NZ
341fca1b 2362 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
f3d096b0
N		 2363 (ANY_EXTRACT:GPI
2364 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2365 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2366 (match_operand 3 "const_int_operand" "n")
2367 (const_int 0)))
2368 (const_int 0)))
2369 (set (match_operand:GPI 0 "register_operand" "=r")
2370 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2371 (mult:GPI (match_dup 1) (match_dup 2))
2372 (match_dup 3)
2373 (const_int 0))))]
2374 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2375 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
99e6af15 2376 [(set_attr "type" "alus_ext")]
f3d096b0
N		 2377)
2378
43e9d192
IB		 2379(define_insn "*add<mode>3nr_compare0"
2380 [(set (reg:CC_NZ CC_REGNUM)
2381 (compare:CC_NZ
34b512f5
SN		 2382 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2383 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
43e9d192
IB		 2384 (const_int 0)))]
2385 ""
2386 "@
2387 cmn\\t%<w>0, %<w>1
e2b691c4 2388 cmn\\t%<w>0, %1
43e9d192 2389 cmp\\t%<w>0, #%n1"
1d61feeb 2390 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 2391)
2392
2f65ab2e
MC		 2393(define_insn "aarch64_sub<mode>_compare0"
2394 [(set (reg:CC_NZ CC_REGNUM)
2395 (compare:CC_NZ
2396 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2397 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2398 (const_int 0)))]
2399 ""
2400 "cmp\\t%<w>0, %<w>1"
2401 [(set_attr "type" "alus_sreg")]
2402)
2403
f5313c63 2404(define_insn "*compare_neg<mode>"
1c992d1e
RE		 2405 [(set (reg:CC_Z CC_REGNUM)
2406 (compare:CC_Z
274b2532
KT		 2407 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2408 (match_operand:GPI 1 "register_operand" "r")))]
f5313c63 2409 ""
274b2532 2410 "cmn\\t%<w>1, %<w>0"
1d61feeb 2411 [(set_attr "type" "alus_sreg")]
f5313c63
IB		 2412)
2413
43e9d192 2414(define_insn "*add_<shift>_<mode>"
24c56925 2415 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 2416 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2417 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2418 (match_operand:GPI 3 "register_operand" "r")))]
2419 ""
2420 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
99e6af15 2421 [(set_attr "type" "alu_shift_imm")]
43e9d192
IB		 2422)
2423
3b585938
IB		 2424;; zero_extend version of above
2425(define_insn "*add_<shift>_si_uxtw"
24c56925 2426 [(set (match_operand:DI 0 "register_operand" "=r")
3b585938
IB		 2427 (zero_extend:DI
2428 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2a9704d0
MS		 2429 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2430 (match_operand:SI 3 "register_operand" "r"))))]
3b585938
IB		 2431 ""
2432 "add\\t%w0, %w3, %w1, <shift> %2"
99e6af15 2433 [(set_attr "type" "alu_shift_imm")]
3b585938
IB		 2434)
2435
43e9d192 2436(define_insn "*add_mul_imm_<mode>"
24c56925 2437 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 2438 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2439 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2440 (match_operand:GPI 3 "register_operand" "r")))]
2441 ""
2442 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
99e6af15 2443 [(set_attr "type" "alu_shift_imm")]
43e9d192
IB		 2444)
2445
2446(define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2447 [(set (match_operand:GPI 0 "register_operand" "=rk")
2448 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2449 (match_operand:GPI 2 "register_operand" "r")))]
2450 ""
2451 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
99e6af15 2452 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2453)
2454
3b585938
IB		 2455;; zero_extend version of above
2456(define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2457 [(set (match_operand:DI 0 "register_operand" "=rk")
2458 (zero_extend:DI
2459 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2460 (match_operand:GPI 2 "register_operand" "r"))))]
2461 ""
2462 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
99e6af15 2463 [(set_attr "type" "alu_ext")]
3b585938
IB		 2464)
2465
43e9d192
IB		 2466(define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2467 [(set (match_operand:GPI 0 "register_operand" "=rk")
2468 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2469 (match_operand:ALLX 1 "register_operand" "r"))
2470 (match_operand 2 "aarch64_imm3" "Ui3"))
2471 (match_operand:GPI 3 "register_operand" "r")))]
2472 ""
2473 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
99e6af15 2474 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2475)
2476
3b585938
IB		 2477;; zero_extend version of above
2478(define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2479 [(set (match_operand:DI 0 "register_operand" "=rk")
2480 (zero_extend:DI
2481 (plus:SI (ashift:SI (ANY_EXTEND:SI
2482 (match_operand:SHORT 1 "register_operand" "r"))
2483 (match_operand 2 "aarch64_imm3" "Ui3"))
2484 (match_operand:SI 3 "register_operand" "r"))))]
2485 ""
2486 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
99e6af15 2487 [(set_attr "type" "alu_ext")]
3b585938
IB		 2488)
2489
43e9d192
IB		 2490(define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2491 [(set (match_operand:GPI 0 "register_operand" "=rk")
2492 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2493 (match_operand:ALLX 1 "register_operand" "r"))
2494 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2495 (match_operand:GPI 3 "register_operand" "r")))]
2496 ""
2497 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
99e6af15 2498 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2499)
2500
3b585938
IB		 2501;; zero_extend version of above
2502(define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2503 [(set (match_operand:DI 0 "register_operand" "=rk")
2504 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2505 (match_operand:SHORT 1 "register_operand" "r"))
2506 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2507 (match_operand:SI 3 "register_operand" "r"))))]
2508 ""
2509 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
99e6af15 2510 [(set_attr "type" "alu_ext")]
3b585938
IB		 2511)
2512
43e9d192
IB		 2513(define_insn "*add_<optab><mode>_multp2"
2514 [(set (match_operand:GPI 0 "register_operand" "=rk")
2515 (plus:GPI (ANY_EXTRACT:GPI
2516 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2517 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2518 (match_operand 3 "const_int_operand" "n")
2519 (const_int 0))
2520 (match_operand:GPI 4 "register_operand" "r")))]
2521 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2522 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
99e6af15 2523 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2524)
2525
3b585938
IB		 2526;; zero_extend version of above
2527(define_insn "*add_<optab>si_multp2_uxtw"
2528 [(set (match_operand:DI 0 "register_operand" "=rk")
2529 (zero_extend:DI
2530 (plus:SI (ANY_EXTRACT:SI
2531 (mult:SI (match_operand:SI 1 "register_operand" "r")
2532 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2533 (match_operand 3 "const_int_operand" "n")
2534 (const_int 0))
2535 (match_operand:SI 4 "register_operand" "r"))))]
2536 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2537 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
99e6af15 2538 [(set_attr "type" "alu_ext")]
3b585938
IB		 2539)
2540
ef22810a
RH		 2541(define_expand "add<mode>3_carryin"
2542 [(set (match_operand:GPI 0 "register_operand")
2543 (plus:GPI
2544 (plus:GPI
f7343f20 2545 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
ef22810a
RH		 2546 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2547 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
3b585938 2548 ""
43e9d192 2549 ""
43e9d192
IB		 2550)
2551
ef22810a
RH		 2552;; Note that add with carry with two zero inputs is matched by cset,
2553;; and that add with carry with one zero input is matched by cinc.
3b585938 2554
ef22810a
RH		 2555(define_insn "*add<mode>3_carryin"
2556 [(set (match_operand:GPI 0 "register_operand" "=r")
2557 (plus:GPI
2558 (plus:GPI
2559 (match_operand:GPI 3 "aarch64_carry_operation" "")
2560 (match_operand:GPI 1 "register_operand" "r"))
2561 (match_operand:GPI 2 "register_operand" "r")))]
43e9d192
IB		 2562 ""
2563 "adc\\t%<w>0, %<w>1, %<w>2"
99e6af15 2564 [(set_attr "type" "adc_reg")]
43e9d192
IB		 2565)
2566
3b585938 2567;; zero_extend version of above
ef22810a
RH		 2568(define_insn "*addsi3_carryin_uxtw"
2569 [(set (match_operand:DI 0 "register_operand" "=r")
2570 (zero_extend:DI
2571 (plus:SI
2572 (plus:SI
2573 (match_operand:SI 3 "aarch64_carry_operation" "")
2574 (match_operand:SI 1 "register_operand" "r"))
2575 (match_operand:SI 2 "register_operand" "r"))))]
3b585938
IB		 2576 ""
2577 "adc\\t%w0, %w1, %w2"
99e6af15 2578 [(set_attr "type" "adc_reg")]
3b585938
IB		 2579)
2580
30c46053
MC		 2581(define_expand "add<mode>3_carryinC"
2582 [(parallel
2583 [(set (match_dup 3)
f7343f20 2584 (compare:CC_ADC
30c46053
MC		 2585 (plus:<DWI>
2586 (plus:<DWI>
2587 (match_dup 4)
2588 (zero_extend:<DWI>
1bbffb87 2589 (match_operand:GPI 1 "register_operand")))
30c46053 2590 (zero_extend:<DWI>
1bbffb87 2591 (match_operand:GPI 2 "register_operand")))
f7343f20 2592 (match_dup 6)))
30c46053
MC		 2593 (set (match_operand:GPI 0 "register_operand")
2594 (plus:GPI
2595 (plus:GPI (match_dup 5) (match_dup 1))
2596 (match_dup 2)))])]
2597 ""
2598{
f7343f20
RE		 2599 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2600 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2601 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2602 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2603 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2604 << GET_MODE_BITSIZE (<MODE>mode),
2605 TImode);
30c46053
MC		 2606})
2607
2608(define_insn "*add<mode>3_carryinC_zero"
f7343f20
RE		 2609 [(set (reg:CC_ADC CC_REGNUM)
2610 (compare:CC_ADC
30c46053
MC		 2611 (plus:<DWI>
2612 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2613 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
f7343f20 2614 (match_operand 4 "const_scalar_int_operand" "")))
16621f0d 2615 (set (match_operand:GPI 0 "register_operand" "=r")
f7343f20
RE		 2616 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2617 (match_dup 1)))]
2618 "rtx_mode_t (operands[4], <DWI>mode)
2619 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
30c46053
MC		 2620 "adcs\\t%<w>0, %<w>1, <w>zr"
2621 [(set_attr "type" "adc_reg")]
2622)
2623
2624(define_insn "*add<mode>3_carryinC"
f7343f20
RE		 2625 [(set (reg:CC_ADC CC_REGNUM)
2626 (compare:CC_ADC
30c46053
MC		 2627 (plus:<DWI>
2628 (plus:<DWI>
2629 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2630 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2631 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
f7343f20 2632 (match_operand 5 "const_scalar_int_operand" "")))
16621f0d 2633 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053 2634 (plus:GPI
f7343f20
RE		 2635 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2636 (match_dup 1))
30c46053 2637 (match_dup 2)))]
f7343f20
RE		 2638 "rtx_mode_t (operands[5], <DWI>mode)
2639 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
30c46053
MC		 2640 "adcs\\t%<w>0, %<w>1, %<w>2"
2641 [(set_attr "type" "adc_reg")]
2642)
2643
2644(define_expand "add<mode>3_carryinV"
2645 [(parallel
2646 [(set (reg:CC_V CC_REGNUM)
2647 (compare:CC_V
2648 (plus:<DWI>
2649 (plus:<DWI>
2650 (match_dup 3)
2651 (sign_extend:<DWI>
1bbffb87 2652 (match_operand:GPI 1 "register_operand")))
30c46053 2653 (sign_extend:<DWI>
1bbffb87 2654 (match_operand:GPI 2 "register_operand")))
30c46053
MC		 2655 (sign_extend:<DWI>
2656 (plus:GPI
2657 (plus:GPI (match_dup 4) (match_dup 1))
2658 (match_dup 2)))))
2659 (set (match_operand:GPI 0 "register_operand")
2660 (plus:GPI
2661 (plus:GPI (match_dup 4) (match_dup 1))
2662 (match_dup 2)))])]
2663 ""
2664{
2665 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
f7343f20
RE		 2666 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2667 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
30c46053
MC		 2668})
2669
2670(define_insn "*add<mode>3_carryinV_zero"
2671 [(set (reg:CC_V CC_REGNUM)
2672 (compare:CC_V
2673 (plus:<DWI>
2674 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2675 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2676 (sign_extend:<DWI>
2677 (plus:GPI
2678 (match_operand:GPI 3 "aarch64_carry_operation" "")
2679 (match_dup 1)))))
16621f0d 2680 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053
MC		 2681 (plus:GPI (match_dup 3) (match_dup 1)))]
2682 ""
2683 "adcs\\t%<w>0, %<w>1, <w>zr"
2684 [(set_attr "type" "adc_reg")]
2685)
2686
2687(define_insn "*add<mode>3_carryinV"
2688 [(set (reg:CC_V CC_REGNUM)
2689 (compare:CC_V
2690 (plus:<DWI>
2691 (plus:<DWI>
2692 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2693 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2694 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2695 (sign_extend:<DWI>
2696 (plus:GPI
2697 (plus:GPI
2698 (match_operand:GPI 4 "aarch64_carry_operation" "")
2699 (match_dup 1))
2700 (match_dup 2)))))
16621f0d 2701 (set (match_operand:GPI 0 "register_operand" "=r")
30c46053
MC		 2702 (plus:GPI
2703 (plus:GPI (match_dup 4) (match_dup 1))
2704 (match_dup 2)))]
2705 ""
2706 "adcs\\t%<w>0, %<w>1, %<w>2"
2707 [(set_attr "type" "adc_reg")]
2708)
2709
dd1bdce4
VK		 2710(define_insn "*add_uxt<mode>_shift2"
2711 [(set (match_operand:GPI 0 "register_operand" "=rk")
2712 (plus:GPI (and:GPI
2713 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2714 (match_operand 2 "aarch64_imm3" "Ui3"))
2715 (match_operand 3 "const_int_operand" "n"))
2716 (match_operand:GPI 4 "register_operand" "r")))]
2717 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2718 "*
2719 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2720 INTVAL (operands[3])));
2721 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2722 [(set_attr "type" "alu_ext")]
2723)
2724
2725;; zero_extend version of above
2726(define_insn "*add_uxtsi_shift2_uxtw"
2727 [(set (match_operand:DI 0 "register_operand" "=rk")
2728 (zero_extend:DI
2729 (plus:SI (and:SI
2730 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2731 (match_operand 2 "aarch64_imm3" "Ui3"))
2732 (match_operand 3 "const_int_operand" "n"))
2733 (match_operand:SI 4 "register_operand" "r"))))]
2734 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2735 "*
2736 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2737 INTVAL (operands[3])));
2738 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2739 [(set_attr "type" "alu_ext")]
2740)
2741
43e9d192
IB		 2742(define_insn "*add_uxt<mode>_multp2"
2743 [(set (match_operand:GPI 0 "register_operand" "=rk")
2744 (plus:GPI (and:GPI
2745 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2746 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2747 (match_operand 3 "const_int_operand" "n"))
2748 (match_operand:GPI 4 "register_operand" "r")))]
2749 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2750 "*
2751 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2752 INTVAL (operands[3])));
2753 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
99e6af15 2754 [(set_attr "type" "alu_ext")]
43e9d192
IB		 2755)
2756
3b585938
IB		 2757;; zero_extend version of above
2758(define_insn "*add_uxtsi_multp2_uxtw"
2759 [(set (match_operand:DI 0 "register_operand" "=rk")
2760 (zero_extend:DI
2761 (plus:SI (and:SI
2762 (mult:SI (match_operand:SI 1 "register_operand" "r")
2763 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2764 (match_operand 3 "const_int_operand" "n"))
2765 (match_operand:SI 4 "register_operand" "r"))))]
2766 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2767 "*
2768 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2769 INTVAL (operands[3])));
2770 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
99e6af15 2771 [(set_attr "type" "alu_ext")]
3b585938
IB		 2772)
2773
43e9d192
IB		 2774(define_insn "subsi3"
2775 [(set (match_operand:SI 0 "register_operand" "=rk")
7eb8fb77
RS		 2776 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2777 (match_operand:SI 2 "register_operand" "r")))]
43e9d192
IB		 2778 ""
2779 "sub\\t%w0, %w1, %w2"
1d61feeb 2780 [(set_attr "type" "alu_sreg")]
43e9d192
IB		 2781)
2782
3b585938
IB		 2783;; zero_extend version of above
2784(define_insn "*subsi3_uxtw"
2785 [(set (match_operand:DI 0 "register_operand" "=rk")
2786 (zero_extend:DI
7eb8fb77 2787 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 2788 (match_operand:SI 2 "register_operand" "r"))))]
2789 ""
2790 "sub\\t%w0, %w1, %w2"
1d61feeb 2791 [(set_attr "type" "alu_sreg")]
3b585938
IB		 2792)
2793
43e9d192 2794(define_insn "subdi3"
4f2962fd 2795 [(set (match_operand:DI 0 "register_operand" "=rk,w")
7eb8fb77
RS		 2796 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2797 (match_operand:DI 2 "register_operand" "r,w")))]
43e9d192
IB		 2798 ""
2799 "@
2800 sub\\t%x0, %x1, %x2
2801 sub\\t%d0, %d1, %d2"
1d61feeb 2802 [(set_attr "type" "alu_sreg, neon_sub")
488461d8 2803 (set_attr "arch" "*,simd")]
43e9d192
IB		 2804)
2805
a58fe3c5 2806(define_expand "subv<GPI:mode>4"
30c46053 2807 [(match_operand:GPI 0 "register_operand")
a58fe3c5
RE		 2808 (match_operand:GPI 1 "register_operand")
2809 (match_operand:GPI 2 "aarch64_plus_operand")
30c46053
MC		 2810 (label_ref (match_operand 3 "" ""))]
2811 ""
2812{
a58fe3c5
RE		 2813 if (CONST_INT_P (operands[2]))
2814 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2815 else
2816 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
30c46053
MC		 2817 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2818
2819 DONE;
2820})
2821
a58fe3c5
RE		 2822(define_insn "subv<GPI:mode>_insn"
2823 [(set (reg:CC_V CC_REGNUM)
2824 (compare:CC_V
2825 (sign_extend:<DWI>
2826 (minus:GPI
2827 (match_operand:GPI 1 "register_operand" "rk")
2828 (match_operand:GPI 2 "register_operand" "r")))
2829 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2830 (sign_extend:<DWI> (match_dup 2)))))
2831 (set (match_operand:GPI 0 "register_operand" "=r")
2832 (minus:GPI (match_dup 1) (match_dup 2)))]
2833 ""
2834 "subs\\t%<w>0, %<w>1, %<w>2"
2835 [(set_attr "type" "alus_sreg")]
2836)
2837
2838(define_insn "subv<GPI:mode>_imm"
2839 [(set (reg:CC_V CC_REGNUM)
2840 (compare:CC_V
2841 (sign_extend:<DWI>
2842 (minus:GPI
2843 (match_operand:GPI 1 "register_operand" "rk,rk")
2844 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2845 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2846 (match_dup 2))))
2847 (set (match_operand:GPI 0 "register_operand" "=r,r")
2848 (minus:GPI (match_dup 1) (match_dup 2)))]
2849 ""
2850 "@
2851 subs\\t%<w>0, %<w>1, %2
2852 adds\\t%<w>0, %<w>1, #%n2"
2853 [(set_attr "type" "alus_sreg")]
2854)
2855
2856(define_expand "negv<GPI:mode>3"
2857 [(match_operand:GPI 0 "register_operand")
2858 (match_operand:GPI 1 "register_operand")
2859 (label_ref (match_operand 2 "" ""))]
2860 ""
2861 {
2862 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2863 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2864
2865 DONE;
2866 }
2867)
2868
2869(define_insn "negv<GPI:mode>_insn"
2870 [(set (reg:CC_V CC_REGNUM)
2871 (compare:CC_V
2872 (sign_extend:<DWI>
2873 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2874 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2875 (set (match_operand:GPI 0 "register_operand" "=r")
2876 (neg:GPI (match_dup 1)))]
2877 ""
2878 "negs\\t%<w>0, %<w>1"
2879 [(set_attr "type" "alus_sreg")]
2880)
2881
2882(define_insn "negv<GPI:mode>_cmp_only"
2883 [(set (reg:CC_V CC_REGNUM)
2884 (compare:CC_V
2885 (sign_extend:<DWI>
2886 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2887 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2888 ""
2889 "negs\\t%<w>zr, %<w>0"
2890 [(set_attr "type" "alus_sreg")]
2891)
2892
2893(define_insn "*cmpv<GPI:mode>_insn"
2894 [(set (reg:CC_V CC_REGNUM)
2895 (compare:CC_V
2896 (sign_extend:<DWI>
2897 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2898 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2899 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2900 (sign_extend:<DWI> (match_dup 1)))))]
2901 ""
2902 "@
2903 cmp\\t%<w>0, %<w>1
2904 cmp\\t%<w>0, %1
2905 cmp\\t%<w>0, #%n1"
2906 [(set_attr "type" "alus_sreg")]
2907)
2908
30c46053
MC		 2909(define_expand "usubv<mode>4"
2910 [(match_operand:GPI 0 "register_operand")
2911 (match_operand:GPI 1 "aarch64_reg_or_zero")
2912 (match_operand:GPI 2 "aarch64_reg_or_zero")
2913 (label_ref (match_operand 3 "" ""))]
2914 ""
2915{
2916 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2917 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2918
2919 DONE;
2920})
2921
2195867f 2922(define_expand "subti3"
1bbffb87
DZ		 2923 [(set (match_operand:TI 0 "register_operand")
2924 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2925 (match_operand:TI 2 "register_operand")))]
2195867f
RH		 2926 ""
2927{
30c46053
MC		 2928 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2929
2930 aarch64_subvti_scratch_regs (operands[1], operands[2],
2931 &low_dest, &op1_low, &op2_low,
2932 &high_dest, &op1_high, &op2_high);
2195867f 2933
30c46053
MC		 2934 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2935 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2195867f 2936
30c46053
MC		 2937 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2938 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2939 DONE;
2940})
2941
2942(define_expand "subvti4"
2943 [(match_operand:TI 0 "register_operand")
a58fe3c5 2944 (match_operand:TI 1 "register_operand")
30c46053
MC		 2945 (match_operand:TI 2 "aarch64_reg_or_imm")
2946 (label_ref (match_operand 3 "" ""))]
2947 ""
2948{
2949 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2950
2951 aarch64_subvti_scratch_regs (operands[1], operands[2],
2952 &low_dest, &op1_low, &op2_low,
2953 &high_dest, &op1_high, &op2_high);
2954 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
a58fe3c5 2955 high_dest, op1_high, op2_high, false);
30c46053
MC		 2956
2957 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2958 DONE;
2959})
2960
2961(define_expand "usubvti4"
2962 [(match_operand:TI 0 "register_operand")
a58fe3c5 2963 (match_operand:TI 1 "register_operand")
30c46053
MC		 2964 (match_operand:TI 2 "aarch64_reg_or_imm")
2965 (label_ref (match_operand 3 "" ""))]
2966 ""
2967{
2968 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2969
2970 aarch64_subvti_scratch_regs (operands[1], operands[2],
2971 &low_dest, &op1_low, &op2_low,
2972 &high_dest, &op1_high, &op2_high);
2973 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
a58fe3c5 2974 high_dest, op1_high, op2_high, true);
30c46053
MC		 2975
2976 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2195867f
RH		 2977 DONE;
2978})
43e9d192 2979
a58fe3c5
RE		 2980(define_expand "negvti3"
2981 [(match_operand:TI 0 "register_operand")
2982 (match_operand:TI 1 "register_operand")
2983 (label_ref (match_operand 2 "" ""))]
2984 ""
2985 {
2986 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2987 gen_lowpart (DImode, operands[1])));
2988 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2989 gen_highpart (DImode, operands[1])));
2990 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2991
2992 DONE;
2993 }
2994)
2995
2996(define_insn "negdi_carryout"
2997 [(set (reg:CC CC_REGNUM)
2998 (compare:CC
2999 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
3000 (set (match_operand:DI 0 "register_operand" "=r")
3001 (neg:DI (match_dup 1)))]
3002 ""
3003 "negs\\t%0, %1"
3004 [(set_attr "type" "alus_sreg")]
3005)
3006
3007(define_insn "negvdi_carryinV"
3008 [(set (reg:CC_V CC_REGNUM)
3009 (compare:CC_V
3010 (neg:TI (plus:TI
3011 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
3012 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
3013 (sign_extend:TI
3014 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3015 (match_dup 1))))))
3016 (set (match_operand:DI 0 "register_operand" "=r")
3017 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3018 (match_dup 1))))]
3019 ""
3020 "ngcs\\t%0, %1"
3021 [(set_attr "type" "alus_sreg")]
3022)
3023
ef22810a 3024(define_insn "*sub<mode>3_compare0"
43e9d192 3025 [(set (reg:CC_NZ CC_REGNUM)
341fca1b 3026 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3027 (match_operand:GPI 2 "register_operand" "r"))
3028 (const_int 0)))
3029 (set (match_operand:GPI 0 "register_operand" "=r")
3030 (minus:GPI (match_dup 1) (match_dup 2)))]
3031 ""
3032 "subs\\t%<w>0, %<w>1, %<w>2"
1d61feeb 3033 [(set_attr "type" "alus_sreg")]
43e9d192
IB		 3034)
3035
3b585938
IB		 3036;; zero_extend version of above
3037(define_insn "*subsi3_compare0_uxtw"
3038 [(set (reg:CC_NZ CC_REGNUM)
341fca1b 3039 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3040 (match_operand:SI 2 "register_operand" "r"))
3041 (const_int 0)))
3042 (set (match_operand:DI 0 "register_operand" "=r")
3043 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3044 ""
3045 "subs\\t%w0, %w1, %w2"
1d61feeb 3046 [(set_attr "type" "alus_sreg")]
3b585938
IB		 3047)
3048
a58fe3c5 3049(define_insn "sub<mode>3_compare1_imm"
30c46053
MC		 3050 [(set (reg:CC CC_REGNUM)
3051 (compare:CC
341fca1b 3052 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
30c46053
MC		 3053 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3054 (set (match_operand:GPI 0 "register_operand" "=r,r")
3055 (plus:GPI
3056 (match_dup 1)
3057 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3058 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3059 "@
a58fe3c5
RE		 3060 subs\\t%<w>0, %<w>1, %2
3061 adds\\t%<w>0, %<w>1, #%n2"
30c46053
MC		 3062 [(set_attr "type" "alus_imm")]
3063)
3064
ef22810a
RH		 3065(define_insn "sub<mode>3_compare1"
3066 [(set (reg:CC CC_REGNUM)
3067 (compare:CC
341fca1b 3068 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
ef22810a
RH		 3069 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3070 (set (match_operand:GPI 0 "register_operand" "=r")
3071 (minus:GPI (match_dup 1) (match_dup 2)))]
3072 ""
3073 "subs\\t%<w>0, %<w>1, %<w>2"
3074 [(set_attr "type" "alus_sreg")]
3075)
3076
e49d8b68 3077(define_peephole2
341fca1b 3078 [(set (match_operand:GPI 0 "aarch64_general_reg")
e49d8b68
KT		 3079 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3080 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3081 (set (reg:CC CC_REGNUM)
3082 (compare:CC
3083 (match_dup 1)
3084 (match_dup 2)))]
3085 "!reg_overlap_mentioned_p (operands[0], operands[1])
3086 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3087 [(const_int 0)]
3088 {
3089 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3090 operands[2]));
3091 DONE;
3092 }
3093)
3094
de47f61f
KT		 3095;; Same as the above peephole but with the compare and minus in
3096;; swapped order. The restriction on overlap between operand 0
3097;; and operands 1 and 2 doesn't apply here.
3098(define_peephole2
3099 [(set (reg:CC CC_REGNUM)
3100 (compare:CC
3101 (match_operand:GPI 1 "aarch64_reg_or_zero")
3102 (match_operand:GPI 2 "aarch64_reg_or_zero")))
341fca1b 3103 (set (match_operand:GPI 0 "aarch64_general_reg")
de47f61f
KT		 3104 (minus:GPI (match_dup 1)
3105 (match_dup 2)))]
3106 ""
3107 [(const_int 0)]
3108 {
3109 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3110 operands[2]));
3111 DONE;
3112 }
3113)
3114
279dc4b4 3115(define_peephole2
341fca1b 3116 [(set (match_operand:GPI 0 "aarch64_general_reg")
279dc4b4 3117 (plus:GPI (match_operand:GPI 1 "register_operand")
a58fe3c5 3118 (match_operand:GPI 2 "aarch64_plus_immediate")))
279dc4b4
KT		 3119 (set (reg:CC CC_REGNUM)
3120 (compare:CC
3121 (match_dup 1)
3122 (match_operand:GPI 3 "const_int_operand")))]
3123 "!reg_overlap_mentioned_p (operands[0], operands[1])
3124 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3125 [(const_int 0)]
3126 {
3127 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
a58fe3c5 3128 operands[3], operands[2]));
279dc4b4
KT		 3129 DONE;
3130 }
3131)
3132
de47f61f
KT		 3133;; Same as the above peephole but with the compare and minus in
3134;; swapped order. The restriction on overlap between operand 0
3135;; and operands 1 doesn't apply here.
3136(define_peephole2
3137 [(set (reg:CC CC_REGNUM)
3138 (compare:CC
3139 (match_operand:GPI 1 "register_operand")
3140 (match_operand:GPI 3 "const_int_operand")))
341fca1b 3141 (set (match_operand:GPI 0 "aarch64_general_reg")
de47f61f 3142 (plus:GPI (match_dup 1)
a58fe3c5 3143 (match_operand:GPI 2 "aarch64_plus_immediate")))]
de47f61f
KT		 3144 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3145 [(const_int 0)]
3146 {
3147 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
a58fe3c5 3148 operands[3], operands[2]));
de47f61f
KT		 3149 DONE;
3150 }
3151)
3152
43e9d192 3153(define_insn "*sub_<shift>_<mode>"
24c56925 3154 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 3155 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3156 (ASHIFT:GPI
3157 (match_operand:GPI 1 "register_operand" "r")
3158 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3159 ""
3160 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
99e6af15 3161 [(set_attr "type" "alu_shift_imm")]
43e9d192
IB		 3162)
3163
3b585938
IB		 3164;; zero_extend version of above
3165(define_insn "*sub_<shift>_si_uxtw"
24c56925 3166 [(set (match_operand:DI 0 "register_operand" "=r")
3b585938
IB		 3167 (zero_extend:DI
3168 (minus:SI (match_operand:SI 3 "register_operand" "r")
3169 (ASHIFT:SI
3170 (match_operand:SI 1 "register_operand" "r")
3171 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3172 ""
3173 "sub\\t%w0, %w3, %w1, <shift> %2"
99e6af15 3174 [(set_attr "type" "alu_shift_imm")]
3b585938
IB		 3175)
3176
43e9d192 3177(define_insn "*sub_mul_imm_<mode>"
24c56925 3178 [(set (match_operand:GPI 0 "register_operand" "=r")
43e9d192
IB		 3179 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3180 (mult:GPI
3181 (match_operand:GPI 1 "register_operand" "r")
3182 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3183 ""
3184 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
99e6af15 3185 [(set_attr "type" "alu_shift_imm")]
43e9d192
IB		 3186)
3187
3b585938
IB		 3188;; zero_extend version of above
3189(define_insn "*sub_mul_imm_si_uxtw"
24c56925 3190 [(set (match_operand:DI 0 "register_operand" "=r")
3b585938
IB		 3191 (zero_extend:DI
3192 (minus:SI (match_operand:SI 3 "register_operand" "r")
3193 (mult:SI
3194 (match_operand:SI 1 "register_operand" "r")
3195 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3196 ""
3197 "sub\\t%w0, %w3, %w1, lsl %p2"
99e6af15 3198 [(set_attr "type" "alu_shift_imm")]
3b585938
IB		 3199)
3200
43e9d192
IB		 3201(define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3202 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3203 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3204 (ANY_EXTEND:GPI
3205 (match_operand:ALLX 2 "register_operand" "r"))))]
3206 ""
3207 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
99e6af15 3208 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3209)
3210
3b585938
IB		 3211;; zero_extend version of above
3212(define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3213 [(set (match_operand:DI 0 "register_operand" "=rk")
3214 (zero_extend:DI
7eb8fb77 3215 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3216 (ANY_EXTEND:SI
3217 (match_operand:SHORT 2 "register_operand" "r")))))]
3218 ""
3219 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
99e6af15 3220 [(set_attr "type" "alu_ext")]
3b585938
IB		 3221)
3222
43e9d192
IB		 3223(define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3224 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3225 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
43e9d192
IB		 3226 (ashift:GPI (ANY_EXTEND:GPI
3227 (match_operand:ALLX 2 "register_operand" "r"))
3228 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3229 ""
3230 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
99e6af15 3231 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3232)
3233
3b585938
IB		 3234;; zero_extend version of above
3235(define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3236 [(set (match_operand:DI 0 "register_operand" "=rk")
3237 (zero_extend:DI
7eb8fb77 3238 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3b585938
IB		 3239 (ashift:SI (ANY_EXTEND:SI
3240 (match_operand:SHORT 2 "register_operand" "r"))
3241 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3242 ""
3243 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
99e6af15 3244 [(set_attr "type" "alu_ext")]
3b585938
IB		 3245)
3246
43e9d192
IB		 3247(define_insn "*sub_<optab><mode>_multp2"
3248 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3249 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
43e9d192
IB		 3250 (ANY_EXTRACT:GPI
3251 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3252 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3253 (match_operand 3 "const_int_operand" "n")
3254 (const_int 0))))]
3255 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3256 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
99e6af15 3257 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3258)
3259
3b585938
IB		 3260;; zero_extend version of above
3261(define_insn "*sub_<optab>si_multp2_uxtw"
3262 [(set (match_operand:DI 0 "register_operand" "=rk")
3263 (zero_extend:DI
7eb8fb77 3264 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3b585938
IB		 3265 (ANY_EXTRACT:SI
3266 (mult:SI (match_operand:SI 1 "register_operand" "r")
3267 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3268 (match_operand 3 "const_int_operand" "n")
3269 (const_int 0)))))]
3270 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3271 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
99e6af15 3272 [(set_attr "type" "alu_ext")]
3b585938
IB		 3273)
3274
ef22810a
RH		 3275;; The hardware description is op1 + ~op2 + C.
3276;; = op1 + (-op2 + 1) + (1 - !C)
3277;; = op1 - op2 - 1 + 1 - !C
3278;; = op1 - op2 - !C.
3279;; We describe the latter.
3280
3281(define_insn "*sub<mode>3_carryin0"
3282 [(set (match_operand:GPI 0 "register_operand" "=r")
3283 (minus:GPI
3284 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3285 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3286 ""
3287 "sbc\\t%<w>0, %<w>1, <w>zr"
3288 [(set_attr "type" "adc_reg")]
3289)
3290
3291;; zero_extend version of the above
3292(define_insn "*subsi3_carryin_uxtw"
3293 [(set (match_operand:DI 0 "register_operand" "=r")
3294 (zero_extend:DI
3295 (minus:SI
3296 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3297 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3298 ""
3299 "sbc\\t%w0, %w1, wzr"
3300 [(set_attr "type" "adc_reg")]
3301)
3302
3303(define_expand "sub<mode>3_carryin"
3304 [(set (match_operand:GPI 0 "register_operand")
3305 (minus:GPI
3306 (minus:GPI
3307 (match_operand:GPI 1 "aarch64_reg_or_zero")
3308 (match_operand:GPI 2 "register_operand"))
3309 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3310 ""
3311 ""
3312)
3313
3314(define_insn "*sub<mode>3_carryin"
3315 [(set (match_operand:GPI 0 "register_operand" "=r")
3316 (minus:GPI
3317 (minus:GPI
3318 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3319 (match_operand:GPI 2 "register_operand" "r"))
3320 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3321
0b8cdc58
IB		 3322 ""
3323 "sbc\\t%<w>0, %<w>1, %<w>2"
99e6af15 3324 [(set_attr "type" "adc_reg")]
0b8cdc58
IB		 3325)
3326
3327;; zero_extend version of the above
3328(define_insn "*subsi3_carryin_uxtw"
ef22810a
RH		 3329 [(set (match_operand:DI 0 "register_operand" "=r")
3330 (zero_extend:DI
3331 (minus:SI
3332 (minus:SI
3333 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3334 (match_operand:SI 2 "register_operand" "r"))
3335 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3336
3337 ""
3338 "sbc\\t%w0, %w1, %w2"
3339 [(set_attr "type" "adc_reg")]
3340)
3341
3342(define_insn "*sub<mode>3_carryin_alt"
3343 [(set (match_operand:GPI 0 "register_operand" "=r")
3344 (minus:GPI
3345 (minus:GPI
3346 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3347 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3348 (match_operand:GPI 2 "register_operand" "r")))]
3349 ""
3350 "sbc\\t%<w>0, %<w>1, %<w>2"
3351 [(set_attr "type" "adc_reg")]
3352)
3353
3354;; zero_extend version of the above
3355(define_insn "*subsi3_carryin_alt_uxtw"
3356 [(set (match_operand:DI 0 "register_operand" "=r")
3357 (zero_extend:DI
3358 (minus:SI
3359 (minus:SI
3360 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3361 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3362 (match_operand:SI 2 "register_operand" "r"))))]
0b8cdc58
IB		 3363 ""
3364 "sbc\\t%w0, %w1, %w2"
99e6af15 3365 [(set_attr "type" "adc_reg")]
0b8cdc58
IB		 3366)
3367
a58fe3c5 3368(define_expand "usub<GPI:mode>3_carryinC"
30c46053
MC		 3369 [(parallel
3370 [(set (reg:CC CC_REGNUM)
3371 (compare:CC
a58fe3c5 3372 (zero_extend:<DWI>
1bbffb87 3373 (match_operand:GPI 1 "aarch64_reg_or_zero"))
30c46053 3374 (plus:<DWI>
a58fe3c5 3375 (zero_extend:<DWI>
1bbffb87 3376 (match_operand:GPI 2 "register_operand"))
30c46053 3377 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
1bbffb87 3378 (set (match_operand:GPI 0 "register_operand")
30c46053
MC		 3379 (minus:GPI
3380 (minus:GPI (match_dup 1) (match_dup 2))
3381 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3382 ""
3383)
3384
a58fe3c5 3385(define_insn "*usub<GPI:mode>3_carryinC_z1"
30c46053
MC		 3386 [(set (reg:CC CC_REGNUM)
3387 (compare:CC
3388 (const_int 0)
3389 (plus:<DWI>
a58fe3c5 3390 (zero_extend:<DWI>
30c46053
MC		 3391 (match_operand:GPI 1 "register_operand" "r"))
3392 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3393 (set (match_operand:GPI 0 "register_operand" "=r")
3394 (minus:GPI
3395 (neg:GPI (match_dup 1))
3396 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3397 ""
3398 "sbcs\\t%<w>0, <w>zr, %<w>1"
3399 [(set_attr "type" "adc_reg")]
3400)
3401
a58fe3c5 3402(define_insn "*usub<GPI:mode>3_carryinC_z2"
30c46053
MC		 3403 [(set (reg:CC CC_REGNUM)
3404 (compare:CC
a58fe3c5 3405 (zero_extend:<DWI>
30c46053
MC		 3406 (match_operand:GPI 1 "register_operand" "r"))
3407 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3408 (set (match_operand:GPI 0 "register_operand" "=r")
3409 (minus:GPI
3410 (match_dup 1)
3411 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3412 ""
3413 "sbcs\\t%<w>0, %<w>1, <w>zr"
3414 [(set_attr "type" "adc_reg")]
3415)
3416
a58fe3c5 3417(define_insn "*usub<GPI:mode>3_carryinC"
30c46053
MC		 3418 [(set (reg:CC CC_REGNUM)
3419 (compare:CC
a58fe3c5 3420 (zero_extend:<DWI>
30c46053
MC		 3421 (match_operand:GPI 1 "register_operand" "r"))
3422 (plus:<DWI>
a58fe3c5 3423 (zero_extend:<DWI>
30c46053
MC		 3424 (match_operand:GPI 2 "register_operand" "r"))
3425 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3426 (set (match_operand:GPI 0 "register_operand" "=r")
3427 (minus:GPI
3428 (minus:GPI (match_dup 1) (match_dup 2))
3429 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3430 ""
3431 "sbcs\\t%<w>0, %<w>1, %<w>2"
3432 [(set_attr "type" "adc_reg")]
3433)
3434
a58fe3c5
RE		 3435(define_expand "sub<GPI:mode>3_carryinV"
3436 [(parallel
3437 [(set (reg:CC_V CC_REGNUM)
3438 (compare:CC_V
3439 (minus:<DWI>
3440 (sign_extend:<DWI>
1bbffb87 3441 (match_operand:GPI 1 "aarch64_reg_or_zero"))
a58fe3c5
RE		 3442 (plus:<DWI>
3443 (sign_extend:<DWI>
1bbffb87 3444 (match_operand:GPI 2 "register_operand"))
a58fe3c5
RE		 3445 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3446 (sign_extend:<DWI>
3447 (minus:GPI (match_dup 1)
3448 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3449 (match_dup 2))))))
1bbffb87 3450 (set (match_operand:GPI 0 "register_operand")
a58fe3c5
RE		 3451 (minus:GPI
3452 (minus:GPI (match_dup 1) (match_dup 2))
3453 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3454 ""
3455)
3456
3457(define_insn "*sub<mode>3_carryinV_z2"
3458 [(set (reg:CC_V CC_REGNUM)
3459 (compare:CC_V
3460 (minus:<DWI>
3461 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3462 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3463 (sign_extend:<DWI>
3464 (minus:GPI (match_dup 1)
3465 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3466 (set (match_operand:GPI 0 "register_operand" "=r")
3467 (minus:GPI
3468 (match_dup 1) (match_dup 3)))]
3469 ""
3470 "sbcs\\t%<w>0, %<w>1, <w>zr"
3471 [(set_attr "type" "adc_reg")]
3472)
3473
3474(define_insn "*sub<mode>3_carryinV"
3475 [(set (reg:CC_V CC_REGNUM)
3476 (compare:CC_V
3477 (minus:<DWI>
3478 (sign_extend:<DWI>
3479 (match_operand:GPI 1 "register_operand" "r"))
3480 (plus:<DWI>
3481 (sign_extend:<DWI>
3482 (match_operand:GPI 2 "register_operand" "r"))
3483 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3484 (sign_extend:<DWI>
3485 (minus:GPI
3486 (match_dup 1)
3487 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3488 (match_dup 2))))))
3489 (set (match_operand:GPI 0 "register_operand" "=r")
3490 (minus:GPI
3491 (minus:GPI (match_dup 1) (match_dup 2))
3492 (match_dup 4)))]
3493 ""
3494 "sbcs\\t%<w>0, %<w>1, %<w>2"
3495 [(set_attr "type" "adc_reg")]
3496)
3497
dd1bdce4
VK		 3498(define_insn "*sub_uxt<mode>_shift2"
3499 [(set (match_operand:GPI 0 "register_operand" "=rk")
3500 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3501 (and:GPI
3502 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3503 (match_operand 2 "aarch64_imm3" "Ui3"))
3504 (match_operand 3 "const_int_operand" "n"))))]
3505 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3506 "*
3507 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3508 INTVAL (operands[3])));
3509 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3510 [(set_attr "type" "alu_ext")]
3511)
3512
3513;; zero_extend version of above
3514(define_insn "*sub_uxtsi_shift2_uxtw"
3515 [(set (match_operand:DI 0 "register_operand" "=rk")
3516 (zero_extend:DI
3517 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3518 (and:SI
3519 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3520 (match_operand 2 "aarch64_imm3" "Ui3"))
3521 (match_operand 3 "const_int_operand" "n")))))]
3522 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3523 "*
3524 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3525 INTVAL (operands[3])));
3526 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3527 [(set_attr "type" "alu_ext")]
3528)
3529
43e9d192
IB		 3530(define_insn "*sub_uxt<mode>_multp2"
3531 [(set (match_operand:GPI 0 "register_operand" "=rk")
7eb8fb77 3532 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
43e9d192
IB		 3533 (and:GPI
3534 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3535 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3536 (match_operand 3 "const_int_operand" "n"))))]
3537 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3538 "*
3539 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3540 INTVAL (operands[3])));
3541 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
99e6af15 3542 [(set_attr "type" "alu_ext")]
43e9d192
IB		 3543)
3544
3b585938
IB		 3545;; zero_extend version of above
3546(define_insn "*sub_uxtsi_multp2_uxtw"
3547 [(set (match_operand:DI 0 "register_operand" "=rk")
3548 (zero_extend:DI
7eb8fb77 3549 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3b585938
IB		 3550 (and:SI
3551 (mult:SI (match_operand:SI 1 "register_operand" "r")
3552 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3553 (match_operand 3 "const_int_operand" "n")))))]
3554 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3555 "*
3556 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3557 INTVAL (operands[3])));
3558 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
99e6af15 3559 [(set_attr "type" "alu_ext")]
3b585938
IB		 3560)
3561
fc8b587d 3562(define_expand "abs<mode>2"
1bbffb87
DZ		 3563 [(match_operand:GPI 0 "register_operand")
3564 (match_operand:GPI 1 "register_operand")]
26366d28 3565 ""
26366d28 3566 {
fc8b587d
WD		 3567 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3568 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3569 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
26366d28
IB		 3570 DONE;
3571 }
26366d28
IB		 3572)
3573
43e9d192 3574(define_insn "neg<mode>2"
040d8a1c
IB		 3575 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3576 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
43e9d192 3577 ""
040d8a1c
IB		 3578 "@
3579 neg\\t%<w>0, %<w>1
3580 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
1d61feeb 3581 [(set_attr "type" "alu_sreg, neon_neg<q>")
488461d8 3582 (set_attr "arch" "*,simd")]
43e9d192
IB		 3583)
3584
3b585938
IB		 3585;; zero_extend version of above
3586(define_insn "*negsi2_uxtw"
3587 [(set (match_operand:DI 0 "register_operand" "=r")
3588 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3589 ""
3590 "neg\\t%w0, %w1"
1d61feeb 3591 [(set_attr "type" "alu_sreg")]
3b585938
IB		 3592)
3593
7e0228bf
N		 3594(define_insn "*ngc<mode>"
3595 [(set (match_operand:GPI 0 "register_operand" "=r")
ef22810a
RH		 3596 (minus:GPI
3597 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3598 (match_operand:GPI 1 "register_operand" "r")))]
7e0228bf
N		 3599 ""
3600 "ngc\\t%<w>0, %<w>1"
99e6af15 3601 [(set_attr "type" "adc_reg")]
7e0228bf
N		 3602)
3603
3604(define_insn "*ngcsi_uxtw"
3605 [(set (match_operand:DI 0 "register_operand" "=r")
3606 (zero_extend:DI
ef22810a
RH		 3607 (minus:SI
3608 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3609 (match_operand:SI 1 "register_operand" "r"))))]
7e0228bf
N		 3610 ""
3611 "ngc\\t%w0, %w1"
99e6af15 3612 [(set_attr "type" "adc_reg")]
7e0228bf
N		 3613)
3614
4f58fe36 3615(define_insn "neg<mode>2_compare0"
43e9d192
IB		 3616 [(set (reg:CC_NZ CC_REGNUM)
3617 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3618 (const_int 0)))
3619 (set (match_operand:GPI 0 "register_operand" "=r")
3620 (neg:GPI (match_dup 1)))]
3621 ""
3622 "negs\\t%<w>0, %<w>1"
1d61feeb 3623 [(set_attr "type" "alus_sreg")]
43e9d192
IB		 3624)
3625
3b585938
IB		 3626;; zero_extend version of above
3627(define_insn "*negsi2_compare0_uxtw"
3628 [(set (reg:CC_NZ CC_REGNUM)
3629 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3630 (const_int 0)))
3631 (set (match_operand:DI 0 "register_operand" "=r")
3632 (zero_extend:DI (neg:SI (match_dup 1))))]
3633 ""
3634 "negs\\t%w0, %w1"
1d61feeb 3635 [(set_attr "type" "alus_sreg")]
3b585938
IB		 3636)
3637
b056c910
N		 3638(define_insn "*neg_<shift><mode>3_compare0"
3639 [(set (reg:CC_NZ CC_REGNUM)
3640 (compare:CC_NZ
3641 (neg:GPI (ASHIFT:GPI
3642 (match_operand:GPI 1 "register_operand" "r")
3643 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3644 (const_int 0)))
3645 (set (match_operand:GPI 0 "register_operand" "=r")
3646 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3647 ""
3648 "negs\\t%<w>0, %<w>1, <shift> %2"
99e6af15 3649 [(set_attr "type" "alus_shift_imm")]
b056c910
N		 3650)
3651
43e9d192
IB		 3652(define_insn "*neg_<shift>_<mode>2"
3653 [(set (match_operand:GPI 0 "register_operand" "=r")
3654 (neg:GPI (ASHIFT:GPI
3655 (match_operand:GPI 1 "register_operand" "r")
3656 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3657 ""
3658 "neg\\t%<w>0, %<w>1, <shift> %2"
99e6af15 3659 [(set_attr "type" "alu_shift_imm")]
43e9d192
IB		 3660)
3661
3b585938
IB		 3662;; zero_extend version of above
3663(define_insn "*neg_<shift>_si2_uxtw"
3664 [(set (match_operand:DI 0 "register_operand" "=r")
3665 (zero_extend:DI
3666 (neg:SI (ASHIFT:SI
3667 (match_operand:SI 1 "register_operand" "r")
3668 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3669 ""
3670 "neg\\t%w0, %w1, <shift> %2"
99e6af15 3671 [(set_attr "type" "alu_shift_imm")]
3b585938
IB		 3672)
3673
43e9d192
IB		 3674(define_insn "*neg_mul_imm_<mode>2"
3675 [(set (match_operand:GPI 0 "register_operand" "=r")
3676 (neg:GPI (mult:GPI
3677 (match_operand:GPI 1 "register_operand" "r")
3678 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3679 ""
3680 "neg\\t%<w>0, %<w>1, lsl %p2"
99e6af15 3681 [(set_attr "type" "alu_shift_imm")]
43e9d192
IB		 3682)
3683
3b585938
IB		 3684;; zero_extend version of above
3685(define_insn "*neg_mul_imm_si2_uxtw"
3686 [(set (match_operand:DI 0 "register_operand" "=r")
3687 (zero_extend:DI
3688 (neg:SI (mult:SI
3689 (match_operand:SI 1 "register_operand" "r")
3690 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3691 ""
3692 "neg\\t%w0, %w1, lsl %p2"
99e6af15 3693 [(set_attr "type" "alu_shift_imm")]
3b585938
IB		 3694)
3695
43e9d192
IB		 3696(define_insn "mul<mode>3"
3697 [(set (match_operand:GPI 0 "register_operand" "=r")
3698 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3699 (match_operand:GPI 2 "register_operand" "r")))]
3700 ""
3701 "mul\\t%<w>0, %<w>1, %<w>2"
99e6af15 3702 [(set_attr "type" "mul")]
43e9d192
IB		 3703)
3704
3b585938
IB		 3705;; zero_extend version of above
3706(define_insn "*mulsi3_uxtw"
3707 [(set (match_operand:DI 0 "register_operand" "=r")
3708 (zero_extend:DI
3709 (mult:SI (match_operand:SI 1 "register_operand" "r")
3710 (match_operand:SI 2 "register_operand" "r"))))]
3711 ""
3712 "mul\\t%w0, %w1, %w2"
99e6af15 3713 [(set_attr "type" "mul")]
3b585938
IB		 3714)
3715
2195867f 3716(define_insn "madd<mode>"
43e9d192
IB		 3717 [(set (match_operand:GPI 0 "register_operand" "=r")
3718 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3719 (match_operand:GPI 2 "register_operand" "r"))
3720 (match_operand:GPI 3 "register_operand" "r")))]
3721 ""
3722 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
99e6af15 3723 [(set_attr "type" "mla")]
43e9d192
IB		 3724)
3725
3b585938
IB		 3726;; zero_extend version of above
3727(define_insn "*maddsi_uxtw"
3728 [(set (match_operand:DI 0 "register_operand" "=r")
3729 (zero_extend:DI
3730 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3731 (match_operand:SI 2 "register_operand" "r"))
3732 (match_operand:SI 3 "register_operand" "r"))))]
3733 ""
3734 "madd\\t%w0, %w1, %w2, %w3"
99e6af15 3735 [(set_attr "type" "mla")]
3b585938
IB		 3736)
3737
43e9d192
IB		 3738(define_insn "*msub<mode>"
3739 [(set (match_operand:GPI 0 "register_operand" "=r")
3740 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3741 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3742 (match_operand:GPI 2 "register_operand" "r"))))]
3743
3744 ""
3745 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
99e6af15 3746 [(set_attr "type" "mla")]
43e9d192
IB		 3747)
3748
3b585938
IB		 3749;; zero_extend version of above
3750(define_insn "*msubsi_uxtw"
3751 [(set (match_operand:DI 0 "register_operand" "=r")
3752 (zero_extend:DI
3753 (minus:SI (match_operand:SI 3 "register_operand" "r")
3754 (mult:SI (match_operand:SI 1 "register_operand" "r")
3755 (match_operand:SI 2 "register_operand" "r")))))]
3756
3757 ""
3758 "msub\\t%w0, %w1, %w2, %w3"
99e6af15 3759 [(set_attr "type" "mla")]
3b585938
IB		 3760)
3761
43e9d192
IB		 3762(define_insn "*mul<mode>_neg"
3763 [(set (match_operand:GPI 0 "register_operand" "=r")
3764 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3765 (match_operand:GPI 2 "register_operand" "r")))]
3766
3767 ""
3768 "mneg\\t%<w>0, %<w>1, %<w>2"
99e6af15 3769 [(set_attr "type" "mul")]
43e9d192
IB		 3770)
3771
3b585938
IB		 3772;; zero_extend version of above
3773(define_insn "*mulsi_neg_uxtw"
3774 [(set (match_operand:DI 0 "register_operand" "=r")
3775 (zero_extend:DI
3776 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3777 (match_operand:SI 2 "register_operand" "r"))))]
3778
3779 ""
3780 "mneg\\t%w0, %w1, %w2"
99e6af15 3781 [(set_attr "type" "mul")]
3b585938
IB		 3782)
3783
43e9d192
IB		 3784(define_insn "<su_optab>mulsidi3"
3785 [(set (match_operand:DI 0 "register_operand" "=r")
3786 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3787 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3788 ""
3789 "<su>mull\\t%0, %w1, %w2"
99e6af15 3790 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3791)
3792
3793(define_insn "<su_optab>maddsidi4"
3794 [(set (match_operand:DI 0 "register_operand" "=r")
3795 (plus:DI (mult:DI
3796 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3797 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3798 (match_operand:DI 3 "register_operand" "r")))]
3799 ""
3800 "<su>maddl\\t%0, %w1, %w2, %3"
99e6af15 3801 [(set_attr "type" "<su>mlal")]
43e9d192
IB		 3802)
3803
3804(define_insn "<su_optab>msubsidi4"
3805 [(set (match_operand:DI 0 "register_operand" "=r")
3806 (minus:DI
3807 (match_operand:DI 3 "register_operand" "r")
3808 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3809 (ANY_EXTEND:DI
3810 (match_operand:SI 2 "register_operand" "r")))))]
3811 ""
3812 "<su>msubl\\t%0, %w1, %w2, %3"
99e6af15 3813 [(set_attr "type" "<su>mlal")]
43e9d192
IB		 3814)
3815
3816(define_insn "*<su_optab>mulsidi_neg"
3817 [(set (match_operand:DI 0 "register_operand" "=r")
3818 (mult:DI (neg:DI
3819 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3820 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3821 ""
3822 "<su>mnegl\\t%0, %w1, %w2"
99e6af15 3823 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3824)
3825
2195867f
RH		 3826(define_expand "<su_optab>mulditi3"
3827 [(set (match_operand:TI 0 "register_operand")
3828 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3829 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3830 ""
3831{
3832 rtx low = gen_reg_rtx (DImode);
3833 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3834
3835 rtx high = gen_reg_rtx (DImode);
3836 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3837
3838 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3839 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3840 DONE;
3841})
3842
3843;; The default expansion of multi3 using umuldi3_highpart will perform
3844;; the additions in an order that fails to combine into two madd insns.
3845(define_expand "multi3"
3846 [(set (match_operand:TI 0 "register_operand")
3847 (mult:TI (match_operand:TI 1 "register_operand")
3848 (match_operand:TI 2 "register_operand")))]
3849 ""
3850{
3851 rtx l0 = gen_reg_rtx (DImode);
3852 rtx l1 = gen_lowpart (DImode, operands[1]);
3853 rtx l2 = gen_lowpart (DImode, operands[2]);
3854 rtx h0 = gen_reg_rtx (DImode);
3855 rtx h1 = gen_highpart (DImode, operands[1]);
3856 rtx h2 = gen_highpart (DImode, operands[2]);
3857
3858 emit_insn (gen_muldi3 (l0, l1, l2));
3859 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3860 emit_insn (gen_madddi (h0, h1, l2, h0));
3861 emit_insn (gen_madddi (h0, l1, h2, h0));
3862
3863 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3864 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3865 DONE;
3866})
3867
43e9d192
IB		 3868(define_insn "<su>muldi3_highpart"
3869 [(set (match_operand:DI 0 "register_operand" "=r")
3870 (truncate:DI
3871 (lshiftrt:TI
3872 (mult:TI
3873 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3874 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3875 (const_int 64))))]
3876 ""
3877 "<su>mulh\\t%0, %1, %2"
99e6af15 3878 [(set_attr "type" "<su>mull")]
43e9d192
IB		 3879)
3880
3881(define_insn "<su_optab>div<mode>3"
3882 [(set (match_operand:GPI 0 "register_operand" "=r")
3883 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3884 (match_operand:GPI 2 "register_operand" "r")))]
3885 ""
3886 "<su>div\\t%<w>0, %<w>1, %<w>2"
99e6af15 3887 [(set_attr "type" "<su>div")]
43e9d192
IB		 3888)
3889
3b585938
IB		 3890;; zero_extend version of above
3891(define_insn "*<su_optab>divsi3_uxtw"
3892 [(set (match_operand:DI 0 "register_operand" "=r")
3893 (zero_extend:DI
3894 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3895 (match_operand:SI 2 "register_operand" "r"))))]
3896 ""
3897 "<su>div\\t%w0, %w1, %w2"
99e6af15 3898 [(set_attr "type" "<su>div")]
3b585938
IB		 3899)
3900
43e9d192
IB		 3901;; -------------------------------------------------------------------
3902;; Comparison insns
3903;; -------------------------------------------------------------------
3904
c8012fbc 3905(define_insn "cmp<mode>"
43e9d192 3906 [(set (reg:CC CC_REGNUM)
3751345d 3907 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
34b512f5 3908 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
43e9d192
IB		 3909 ""
3910 "@
34b512f5 3911 cmp\\t%<w>0, %<w>1
e2b691c4 3912 cmp\\t%<w>0, %1
43e9d192 3913 cmn\\t%<w>0, #%n1"
1d61feeb 3914 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
43e9d192
IB		 3915)
3916
786e3c06 3917(define_insn "fcmp<mode>"
43e9d192
IB		 3918 [(set (reg:CCFP CC_REGNUM)
3919 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3920 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3921 "TARGET_FLOAT"
3922 "@
3923 fcmp\\t%<s>0, #0.0
3924 fcmp\\t%<s>0, %<s>1"
99e6af15 3925 [(set_attr "type" "fcmp<s>")]
43e9d192
IB		 3926)
3927
786e3c06 3928(define_insn "fcmpe<mode>"
43e9d192
IB		 3929 [(set (reg:CCFPE CC_REGNUM)
3930 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3931 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3932 "TARGET_FLOAT"
3933 "@
3934 fcmpe\\t%<s>0, #0.0
3935 fcmpe\\t%<s>0, %<s>1"
99e6af15 3936 [(set_attr "type" "fcmp<s>")]
43e9d192
IB		 3937)
3938
3939(define_insn "*cmp_swp_<shift>_reg<mode>"
3940 [(set (reg:CC_SWP CC_REGNUM)
3941 (compare:CC_SWP (ASHIFT:GPI
3942 (match_operand:GPI 0 "register_operand" "r")
3943 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3944 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3945 ""
3946 "cmp\\t%<w>2, %<w>0, <shift> %1"
99e6af15 3947 [(set_attr "type" "alus_shift_imm")]
43e9d192
IB		 3948)
3949
3950(define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3951 [(set (reg:CC_SWP CC_REGNUM)
3952 (compare:CC_SWP (ANY_EXTEND:GPI
3953 (match_operand:ALLX 0 "register_operand" "r"))
3954 (match_operand:GPI 1 "register_operand" "r")))]
3955 ""
3956 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
99e6af15 3957 [(set_attr "type" "alus_ext")]
43e9d192
IB		 3958)
3959
58cff58c
N		 3960(define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3961 [(set (reg:CC_SWP CC_REGNUM)
3962 (compare:CC_SWP (ashift:GPI
3963 (ANY_EXTEND:GPI
3964 (match_operand:ALLX 0 "register_operand" "r"))
4095f9fa 3965 (match_operand 1 "aarch64_imm3" "Ui3"))
58cff58c
N		 3966 (match_operand:GPI 2 "register_operand" "r")))]
3967 ""
3968 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
99e6af15 3969 [(set_attr "type" "alus_ext")]
58cff58c 3970)
43e9d192
IB		 3971
3972;; -------------------------------------------------------------------
3973;; Store-flag and conditional select insns
3974;; -------------------------------------------------------------------
3975
3976(define_expand "cstore<mode>4"
1bbffb87 3977 [(set (match_operand:SI 0 "register_operand")
43e9d192 3978 (match_operator:SI 1 "aarch64_comparison_operator"
1bbffb87
DZ		 3979 [(match_operand:GPI 2 "register_operand")
3980 (match_operand:GPI 3 "aarch64_plus_operand")]))]
43e9d192
IB		 3981 ""
3982 "
3983 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3984 operands[3]);
3985 operands[3] = const0_rtx;
3986 "
3987)
3988
cf670503
ZC		 3989(define_expand "cstorecc4"
3990 [(set (match_operand:SI 0 "register_operand")
a44e0ebb 3991 (match_operator 1 "aarch64_comparison_operator_mode"
c8012fbc 3992 [(match_operand 2 "cc_register")
cf670503
ZC		 3993 (match_operand 3 "const0_operand")]))]
3994 ""
3995"{
f7df4a84 3996 emit_insn (gen_rtx_SET (operands[0], operands[1]));
cf670503
ZC		 3997 DONE;
3998}")
3999
4000
43e9d192 4001(define_expand "cstore<mode>4"
1bbffb87 4002 [(set (match_operand:SI 0 "register_operand")
a44e0ebb 4003 (match_operator:SI 1 "aarch64_comparison_operator_mode"
1bbffb87
DZ		 4004 [(match_operand:GPF 2 "register_operand")
4005 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
43e9d192
IB		 4006 ""
4007 "
4008 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4009 operands[3]);
4010 operands[3] = const0_rtx;
4011 "
4012)
4013
714e1b3b 4014(define_insn "aarch64_cstore<mode>"
43e9d192 4015 [(set (match_operand:ALLI 0 "register_operand" "=r")
a44e0ebb 4016 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
43e9d192
IB		 4017 [(match_operand 2 "cc_register" "") (const_int 0)]))]
4018 ""
4019 "cset\\t%<w>0, %m1"
99e6af15 4020 [(set_attr "type" "csel")]
43e9d192
IB		 4021)
4022
714e1b3b
KT		 4023;; For a 24-bit immediate CST we can optimize the compare for equality
4024;; and branch sequence from:
4025;; mov x0, #imm1
4026;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
4027;; cmp x1, x0
4028;; cset x2, <ne,eq>
4029;; into the shorter:
4030;; sub x0, x1, #(CST & 0xfff000)
4031;; subs x0, x0, #(CST & 0x000fff)
4032;; cset x2, <ne, eq>.
4033(define_insn_and_split "*compare_cstore<mode>_insn"
4034 [(set (match_operand:GPI 0 "register_operand" "=r")
4035 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
c8574943
KT		 4036 (match_operand:GPI 2 "aarch64_imm24" "n")))
4037 (clobber (reg:CC CC_REGNUM))]
714e1b3b
KT		 4038 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4039 && !aarch64_plus_operand (operands[2], <MODE>mode)
4040 && !reload_completed"
4041 "#"
4042 "&& true"
4043 [(const_int 0)]
4044 {
4045 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4046 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4047 rtx tmp = gen_reg_rtx (<MODE>mode);
4048 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4049 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4050 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4051 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4052 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4053 DONE;
4054 }
4055 [(set_attr "type" "csel")]
4056)
4057
d0b6bb1b
IB		 4058;; zero_extend version of the above
4059(define_insn "*cstoresi_insn_uxtw"
4060 [(set (match_operand:DI 0 "register_operand" "=r")
4061 (zero_extend:DI
a44e0ebb 4062 (match_operator:SI 1 "aarch64_comparison_operator_mode"
d0b6bb1b
IB		 4063 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4064 ""
4065 "cset\\t%w0, %m1"
99e6af15 4066 [(set_attr "type" "csel")]
d0b6bb1b
IB		 4067)
4068
889b9412 4069(define_insn "cstore<mode>_neg"
43e9d192 4070 [(set (match_operand:ALLI 0 "register_operand" "=r")
a44e0ebb 4071 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
43e9d192
IB		 4072 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4073 ""
4074 "csetm\\t%<w>0, %m1"
99e6af15 4075 [(set_attr "type" "csel")]
43e9d192
IB		 4076)
4077
d0b6bb1b
IB		 4078;; zero_extend version of the above
4079(define_insn "*cstoresi_neg_uxtw"
4080 [(set (match_operand:DI 0 "register_operand" "=r")
4081 (zero_extend:DI
a44e0ebb 4082 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
d0b6bb1b
IB		 4083 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4084 ""
4085 "csetm\\t%w0, %m1"
99e6af15 4086 [(set_attr "type" "csel")]
d0b6bb1b
IB		 4087)
4088
43e9d192 4089(define_expand "cmov<mode>6"
1bbffb87 4090 [(set (match_operand:GPI 0 "register_operand")
43e9d192
IB		 4091 (if_then_else:GPI
4092 (match_operator 1 "aarch64_comparison_operator"
1bbffb87
DZ		 4093 [(match_operand:GPI 2 "register_operand")
4094 (match_operand:GPI 3 "aarch64_plus_operand")])
4095 (match_operand:GPI 4 "register_operand")
4096 (match_operand:GPI 5 "register_operand")))]
43e9d192
IB		 4097 ""
4098 "
4099 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4100 operands[3]);
4101 operands[3] = const0_rtx;
4102 "
4103)
4104
4105(define_expand "cmov<mode>6"
1bbffb87 4106 [(set (match_operand:GPF 0 "register_operand")
43e9d192
IB		 4107 (if_then_else:GPF
4108 (match_operator 1 "aarch64_comparison_operator"
1bbffb87
DZ		 4109 [(match_operand:GPF 2 "register_operand")
4110 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4111 (match_operand:GPF 4 "register_operand")
4112 (match_operand:GPF 5 "register_operand")))]
43e9d192
IB		 4113 ""
4114 "
4115 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4116 operands[3]);
4117 operands[3] = const0_rtx;
4118 "
4119)
4120
4121(define_insn "*cmov<mode>_insn"
06b1198f 4122 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
43e9d192
IB		 4123 (if_then_else:ALLI
4124 (match_operator 1 "aarch64_comparison_operator"
4125 [(match_operand 2 "cc_register" "") (const_int 0)])
06b1198f
IB		 4126 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4127 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4128 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4129 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4130 ;; Final two alternatives should be unreachable, but included for completeness
43e9d192
IB		 4131 "@
4132 csel\\t%<w>0, %<w>3, %<w>4, %m1
4133 csinv\\t%<w>0, %<w>3, <w>zr, %m1
4134 csinv\\t%<w>0, %<w>4, <w>zr, %M1
06b1198f
IB		 4135 csinc\\t%<w>0, %<w>3, <w>zr, %m1
4136 csinc\\t%<w>0, %<w>4, <w>zr, %M1
4137 mov\\t%<w>0, -1
4138 mov\\t%<w>0, 1"
04e34811 4139 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
43e9d192
IB		 4140)
4141
d0b6bb1b
IB		 4142;; zero_extend version of above
4143(define_insn "*cmovsi_insn_uxtw"
4144 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4145 (zero_extend:DI
4146 (if_then_else:SI
4147 (match_operator 1 "aarch64_comparison_operator"
4148 [(match_operand 2 "cc_register" "") (const_int 0)])
4149 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4150 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4151 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4152 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4153 ;; Final two alternatives should be unreachable, but included for completeness
4154 "@
4155 csel\\t%w0, %w3, %w4, %m1
4156 csinv\\t%w0, %w3, wzr, %m1
4157 csinv\\t%w0, %w4, wzr, %M1
4158 csinc\\t%w0, %w3, wzr, %m1
4159 csinc\\t%w0, %w4, wzr, %M1
4160 mov\\t%w0, -1
4161 mov\\t%w0, 1"
04e34811 4162 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
d0b6bb1b
IB		 4163)
4164
1d3e6ed3
RL		 4165(define_insn "*cmovdi_insn_uxtw"
4166 [(set (match_operand:DI 0 "register_operand" "=r")
4167 (if_then_else:DI
4168 (match_operator 1 "aarch64_comparison_operator"
4169 [(match_operand 2 "cc_register" "") (const_int 0)])
4170 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4171 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4172 ""
4173 "csel\\t%w0, %w3, %w4, %m1"
4174 [(set_attr "type" "csel")]
4175)
4176
43e9d192
IB		 4177(define_insn "*cmov<mode>_insn"
4178 [(set (match_operand:GPF 0 "register_operand" "=w")
4179 (if_then_else:GPF
4180 (match_operator 1 "aarch64_comparison_operator"
4181 [(match_operand 2 "cc_register" "") (const_int 0)])
4182 (match_operand:GPF 3 "register_operand" "w")
4183 (match_operand:GPF 4 "register_operand" "w")))]
4184 "TARGET_FLOAT"
4185 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
99e6af15 4186 [(set_attr "type" "fcsel")]
43e9d192
IB		 4187)
4188
4189(define_expand "mov<mode>cc"
1bbffb87
DZ		 4190 [(set (match_operand:ALLI 0 "register_operand")
4191 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4192 (match_operand:ALLI 2 "register_operand")
4193 (match_operand:ALLI 3 "register_operand")))]
43e9d192
IB		 4194 ""
4195 {
c8012fbc 4196 rtx ccreg;
43e9d192
IB		 4197 enum rtx_code code = GET_CODE (operands[1]);
4198
4199 if (code == UNEQ || code == LTGT)
4200 FAIL;
4201
c8012fbc
WD		 4202 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4203 XEXP (operands[1], 1));
4204 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
43e9d192
IB		 4205 }
4206)
4207
4208(define_expand "mov<GPF:mode><GPI:mode>cc"
1bbffb87
DZ		 4209 [(set (match_operand:GPI 0 "register_operand")
4210 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4211 (match_operand:GPF 2 "register_operand")
4212 (match_operand:GPF 3 "register_operand")))]
43e9d192
IB		 4213 ""
4214 {
4215 rtx ccreg;
4216 enum rtx_code code = GET_CODE (operands[1]);
4217
4218 if (code == UNEQ || code == LTGT)
4219 FAIL;
4220
4221 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4222 XEXP (operands[1], 1));
4223 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4224 }
4225)
4226
d3a3182b 4227(define_expand "mov<mode>cc"
1bbffb87
DZ		 4228 [(set (match_operand:GPF 0 "register_operand")
4229 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4230 (match_operand:GPF 2 "register_operand")
4231 (match_operand:GPF 3 "register_operand")))]
d3a3182b
ZC		 4232 ""
4233 {
4234 rtx ccreg;
4235 enum rtx_code code = GET_CODE (operands[1]);
4236
4237 if (code == UNEQ || code == LTGT)
4238 FAIL;
4239
4240 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4241 XEXP (operands[1], 1));
4242 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4243 }
4244)
4245
3204ac98 4246(define_expand "<neg_not_op><mode>cc"
1bbffb87
DZ		 4247 [(set (match_operand:GPI 0 "register_operand")
4248 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4249 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4250 (match_operand:GPI 3 "register_operand")))]
3204ac98
KT		 4251 ""
4252 {
4253 rtx ccreg;
4254 enum rtx_code code = GET_CODE (operands[1]);
4255
4256 if (code == UNEQ || code == LTGT)
4257 FAIL;
4258
4259 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4260 XEXP (operands[1], 1));
4261 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4262 }
4263)
5d357f26
KT		 4264
4265;; CRC32 instructions.
4266(define_insn "aarch64_<crc_variant>"
4267 [(set (match_operand:SI 0 "register_operand" "=r")
4268 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4269 (match_operand:<crc_mode> 2 "register_operand" "r")]
4270 CRC))]
4271 "TARGET_CRC32"
4272 {
6a70badb 4273 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
5d357f26
KT		 4274 return "<crc_variant>\\t%w0, %w1, %x2";
4275 else
4276 return "<crc_variant>\\t%w0, %w1, %w2";
4277 }
4278 [(set_attr "type" "crc")]
4279)
4280
43e9d192
IB		 4281(define_insn "*csinc2<mode>_insn"
4282 [(set (match_operand:GPI 0 "register_operand" "=r")
cd5660ab
KT		 4283 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4284 (match_operand:GPI 1 "register_operand" "r")))]
43e9d192 4285 ""
49f88e43 4286 "cinc\\t%<w>0, %<w>1, %m2"
99e6af15
JG		 4287 [(set_attr "type" "csel")]
4288)
43e9d192
IB		 4289
4290(define_insn "csinc3<mode>_insn"
4291 [(set (match_operand:GPI 0 "register_operand" "=r")
4292 (if_then_else:GPI
cd5660ab
KT		 4293 (match_operand 1 "aarch64_comparison_operation" "")
4294 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
43e9d192 4295 (const_int 1))
cd5660ab 4296 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4297 ""
cd5660ab 4298 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15 4299 [(set_attr "type" "csel")]
43e9d192
IB		 4300)
4301
4302(define_insn "*csinv3<mode>_insn"
4303 [(set (match_operand:GPI 0 "register_operand" "=r")
4304 (if_then_else:GPI
cd5660ab
KT		 4305 (match_operand 1 "aarch64_comparison_operation" "")
4306 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4307 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4308 ""
cd5660ab 4309 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15
JG		 4310 [(set_attr "type" "csel")]
4311)
43e9d192 4312
0f6ca79c
RL		 4313(define_insn "csneg3_uxtw_insn"
4314 [(set (match_operand:DI 0 "register_operand" "=r")
4315 (zero_extend:DI
4316 (if_then_else:SI
4317 (match_operand 1 "aarch64_comparison_operation" "")
4318 (neg:SI (match_operand:SI 2 "register_operand" "r"))
4319 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4320 ""
4321 "csneg\\t%w0, %w3, %w2, %M1"
4322 [(set_attr "type" "csel")]
4323)
4324
fc8b587d 4325(define_insn "csneg3<mode>_insn"
43e9d192
IB		 4326 [(set (match_operand:GPI 0 "register_operand" "=r")
4327 (if_then_else:GPI
cd5660ab
KT		 4328 (match_operand 1 "aarch64_comparison_operation" "")
4329 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4330 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
43e9d192 4331 ""
cd5660ab 4332 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
99e6af15
JG		 4333 [(set_attr "type" "csel")]
4334)
43e9d192 4335
7cfb4d93
RS		 4336;; If X can be loaded by a single CNT[BHWD] instruction,
4337;;
4338;; A = UMAX (B, X)
4339;;
4340;; is equivalent to:
4341;;
4342;; TMP = UQDEC[BHWD] (B, X)
4343;; A = TMP + X
4344;;
4345;; Defining the pattern this way means that:
4346;;
4347;; A = UMAX (B, X) - X
4348;;
4349;; becomes:
4350;;
4351;; TMP1 = UQDEC[BHWD] (B, X)
4352;; TMP2 = TMP1 + X
4353;; A = TMP2 - X
4354;;
4355;; which combine can optimize to:
4356;;
4357;; A = UQDEC[BHWD] (B, X)
4358;;
4359;; We don't use match_operand predicates because the order of the operands
4360;; can vary: the CNT[BHWD] constant will come first if the other operand is
4361;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4362(define_expand "umax<mode>3"
4363 [(set (match_operand:GPI 0 "register_operand")
4364 (umax:GPI (match_operand:GPI 1 "")
4365 (match_operand:GPI 2 "")))]
4366 "TARGET_SVE"
4367 {
4368 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4369 std::swap (operands[1], operands[2]);
4370 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4371 FAIL;
4372 rtx temp = gen_reg_rtx (<MODE>mode);
4373 operands[1] = force_reg (<MODE>mode, operands[1]);
4374 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4375 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4376 DONE;
4377 }
4378)
4379
4380;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4381(define_insn "aarch64_uqdec<mode>"
4382 [(set (match_operand:GPI 0 "register_operand" "=r")
4383 (minus:GPI
4384 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4385 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4386 (match_dup 2)))]
4387 "TARGET_SVE"
4388 {
4389 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4390 }
4391)
4392
43e9d192
IB		 4393;; -------------------------------------------------------------------
4394;; Logical operations
4395;; -------------------------------------------------------------------
4396
43fd192f
MC		 4397
4398(define_insn_and_split "*aarch64_and<mode>_imm2"
4399 [(set (match_operand:GPI 0 "register_operand" "=rk")
4400 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4401 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4402 ""
4403 "#"
4404 "true"
4405 [(const_int 0)]
4406 {
4407 HOST_WIDE_INT val = INTVAL (operands[2]);
4408 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4409 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4410
4411 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4412 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4413 DONE;
4414 }
4415)
4416
43e9d192 4417(define_insn "<optab><mode>3"
fe82d1f2
AL		 4418 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4419 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4420 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
43e9d192 4421 ""
fe82d1f2
AL		 4422 "@
4423 <logical>\\t%<w>0, %<w>1, %<w>2
e2b691c4 4424 <logical>\\t%<w>0, %<w>1, %2
fe82d1f2
AL		 4425 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4426 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
488461d8 4427 (set_attr "arch" "*,*,simd")]
99e6af15 4428)
43e9d192 4429
d0b6bb1b
IB		 4430;; zero_extend version of above
4431(define_insn "*<optab>si3_uxtw"
4432 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4433 (zero_extend:DI
4434 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4435 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4436 ""
954224d7
WD		 4437 "@
4438 <logical>\\t%w0, %w1, %w2
4439 <logical>\\t%w0, %w1, %2"
99e6af15
JG		 4440 [(set_attr "type" "logic_reg,logic_imm")]
4441)
d0b6bb1b 4442
2a1d1991
N		 4443(define_insn "*and<mode>3_compare0"
4444 [(set (reg:CC_NZ CC_REGNUM)
4445 (compare:CC_NZ
4446 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4447 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4448 (const_int 0)))
4449 (set (match_operand:GPI 0 "register_operand" "=r,r")
4450 (and:GPI (match_dup 1) (match_dup 2)))]
4451 ""
954224d7
WD		 4452 "@
4453 ands\\t%<w>0, %<w>1, %<w>2
4454 ands\\t%<w>0, %<w>1, %2"
99e6af15 4455 [(set_attr "type" "logics_reg,logics_imm")]
2a1d1991
N		 4456)
4457
4458;; zero_extend version of above
4459(define_insn "*andsi3_compare0_uxtw"
4460 [(set (reg:CC_NZ CC_REGNUM)
4461 (compare:CC_NZ
4462 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4463 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4464 (const_int 0)))
4465 (set (match_operand:DI 0 "register_operand" "=r,r")
4466 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4467 ""
954224d7
WD		 4468 "@
4469 ands\\t%w0, %w1, %w2
4470 ands\\t%w0, %w1, %2"
99e6af15 4471 [(set_attr "type" "logics_reg,logics_imm")]
2a1d1991
N		 4472)
4473
4474(define_insn "*and_<SHIFT:optab><mode>3_compare0"
4475 [(set (reg:CC_NZ CC_REGNUM)
4476 (compare:CC_NZ
4477 (and:GPI (SHIFT:GPI
4478 (match_operand:GPI 1 "register_operand" "r")
4479 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4480 (match_operand:GPI 3 "register_operand" "r"))
4481 (const_int 0)))
4482 (set (match_operand:GPI 0 "register_operand" "=r")
4483 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4484 ""
4485 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15 4486 [(set_attr "type" "logics_shift_imm")]
2a1d1991
N		 4487)
4488
4489;; zero_extend version of above
4490(define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4491 [(set (reg:CC_NZ CC_REGNUM)
4492 (compare:CC_NZ
4493 (and:SI (SHIFT:SI
4494 (match_operand:SI 1 "register_operand" "r")
4495 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4496 (match_operand:SI 3 "register_operand" "r"))
4497 (const_int 0)))
4498 (set (match_operand:DI 0 "register_operand" "=r")
4499 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4500 (match_dup 3))))]
4501 ""
4502 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15 4503 [(set_attr "type" "logics_shift_imm")]
2a1d1991
N		 4504)
4505
43e9d192
IB		 4506(define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4507 [(set (match_operand:GPI 0 "register_operand" "=r")
4508 (LOGICAL:GPI (SHIFT:GPI
4509 (match_operand:GPI 1 "register_operand" "r")
4510 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4511 (match_operand:GPI 3 "register_operand" "r")))]
4512 ""
4513 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15
JG		 4514 [(set_attr "type" "logic_shift_imm")]
4515)
43e9d192 4516
57b77d46
RE		 4517(define_insn "*<optab>_rol<mode>3"
4518 [(set (match_operand:GPI 0 "register_operand" "=r")
4519 (LOGICAL:GPI (rotate:GPI
4520 (match_operand:GPI 1 "register_operand" "r")
4521 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4522 (match_operand:GPI 3 "register_operand" "r")))]
4523 ""
4524 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4525 [(set_attr "type" "logic_shift_imm")]
4526)
4527
4528;; zero_extend versions of above
d0b6bb1b
IB		 4529(define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4530 [(set (match_operand:DI 0 "register_operand" "=r")
4531 (zero_extend:DI
4532 (LOGICAL:SI (SHIFT:SI
4533 (match_operand:SI 1 "register_operand" "r")
4534 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4535 (match_operand:SI 3 "register_operand" "r"))))]
4536 ""
4537 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15
JG		 4538 [(set_attr "type" "logic_shift_imm")]
4539)
d0b6bb1b 4540
57b77d46
RE		 4541(define_insn "*<optab>_rolsi3_uxtw"
4542 [(set (match_operand:DI 0 "register_operand" "=r")
4543 (zero_extend:DI
4544 (LOGICAL:SI (rotate:SI
4545 (match_operand:SI 1 "register_operand" "r")
4546 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4547 (match_operand:SI 3 "register_operand" "r"))))]
4548 ""
4549 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4550 [(set_attr "type" "logic_shift_imm")]
4551)
4552
43e9d192 4553(define_insn "one_cmpl<mode>2"
fe82d1f2
AL		 4554 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4555 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
43e9d192 4556 ""
fe82d1f2
AL		 4557 "@
4558 mvn\\t%<w>0, %<w>1
4559 mvn\\t%0.8b, %1.8b"
4560 [(set_attr "type" "logic_reg,neon_logic")
488461d8 4561 (set_attr "arch" "*,simd")]
99e6af15 4562)
43e9d192
IB		 4563
4564(define_insn "*one_cmpl_<optab><mode>2"
4565 [(set (match_operand:GPI 0 "register_operand" "=r")
4566 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4567 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4568 ""
4569 "mvn\\t%<w>0, %<w>1, <shift> %2"
99e6af15
JG		 4570 [(set_attr "type" "logic_shift_imm")]
4571)
43e9d192 4572
84be6032
AL		 4573;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4574
4575(define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4576 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4577 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4578 (match_operand:GPI 2 "register_operand" "r,w")))]
43e9d192 4579 ""
84be6032
AL		 4580 "@
4581 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4582 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4583 [(set_attr "type" "logic_reg,neon_logic")
488461d8 4584 (set_attr "arch" "*,simd")]
84be6032
AL		 4585)
4586
8409e468
KT		 4587(define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4588 [(set (match_operand:DI 0 "register_operand" "=r")
4589 (zero_extend:DI
4590 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4591 (match_operand:SI 2 "register_operand" "r"))))]
4592 ""
4593 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4594 [(set_attr "type" "logic_reg")]
4595)
4596
4597(define_insn "*xor_one_cmplsidi3_ze"
4598 [(set (match_operand:DI 0 "register_operand" "=r")
4599 (zero_extend:DI
4600 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4601 (match_operand:SI 2 "register_operand" "r")))))]
4602 ""
4603 "eon\\t%w0, %w1, %w2"
4604 [(set_attr "type" "logic_reg")]
4605)
4606
84be6032
AL		 4607;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4608;; eon does not operate on SIMD registers so the vector variant must be split.
4609(define_insn_and_split "*xor_one_cmpl<mode>3"
4610 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4611 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4612 (match_operand:GPI 2 "register_operand" "r,w"))))]
4613 ""
eeaccc07
AL		 4614 "@
4615 eon\\t%<w>0, %<w>1, %<w>2
4616 #"
4617 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
84be6032
AL		 4618 [(set (match_operand:GPI 0 "register_operand" "=w")
4619 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4620 (match_operand:GPI 2 "register_operand" "w")))
4621 (set (match_dup 0) (not:GPI (match_dup 0)))]
4622 ""
4623 [(set_attr "type" "logic_reg,multiple")
488461d8 4624 (set_attr "arch" "*,simd")]
99e6af15 4625)
43e9d192 4626
add3c965
IB		 4627(define_insn "*and_one_cmpl<mode>3_compare0"
4628 [(set (reg:CC_NZ CC_REGNUM)
4629 (compare:CC_NZ
4630 (and:GPI (not:GPI
4631 (match_operand:GPI 1 "register_operand" "r"))
4632 (match_operand:GPI 2 "register_operand" "r"))
4633 (const_int 0)))
4634 (set (match_operand:GPI 0 "register_operand" "=r")
4635 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4636 ""
4637 "bics\\t%<w>0, %<w>2, %<w>1"
99e6af15
JG		 4638 [(set_attr "type" "logics_reg")]
4639)
add3c965
IB		 4640
4641;; zero_extend version of above
4642(define_insn "*and_one_cmplsi3_compare0_uxtw"
4643 [(set (reg:CC_NZ CC_REGNUM)
4644 (compare:CC_NZ
4645 (and:SI (not:SI
4646 (match_operand:SI 1 "register_operand" "r"))
4647 (match_operand:SI 2 "register_operand" "r"))
4648 (const_int 0)))
4649 (set (match_operand:DI 0 "register_operand" "=r")
4650 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4651 ""
4652 "bics\\t%w0, %w2, %w1"
99e6af15
JG		 4653 [(set_attr "type" "logics_reg")]
4654)
add3c965 4655
cd8fc5ee
AV		 4656(define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4657 [(set (reg:CC_NZ CC_REGNUM)
4658 (compare:CC_NZ
4659 (and:GPI (not:GPI
4660 (match_operand:GPI 0 "register_operand" "r"))
4661 (match_operand:GPI 1 "register_operand" "r"))
4662 (const_int 0)))]
4663 ""
4664 "bics\\t<w>zr, %<w>1, %<w>0"
4665 [(set_attr "type" "logics_reg")]
4666)
4667
cc05c2c3 4668(define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
43e9d192
IB		 4669 [(set (match_operand:GPI 0 "register_operand" "=r")
4670 (LOGICAL:GPI (not:GPI
4671 (SHIFT:GPI
4672 (match_operand:GPI 1 "register_operand" "r")
4673 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4674 (match_operand:GPI 3 "register_operand" "r")))]
4675 ""
4676 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
dba606bf 4677 [(set_attr "type" "logic_shift_imm")]
99e6af15 4678)
43e9d192 4679
6365da9e
KT		 4680(define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4681 [(set (match_operand:GPI 0 "register_operand" "=r")
4682 (not:GPI (xor:GPI
4683 (SHIFT:GPI
4684 (match_operand:GPI 1 "register_operand" "r")
4685 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4686 (match_operand:GPI 3 "register_operand" "r"))))]
4687 ""
4688 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4689 [(set_attr "type" "logic_shift_imm")]
4690)
4691
4692;; Zero-extend version of the above.
4693(define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4694 [(set (match_operand:DI 0 "register_operand" "=r")
4695 (zero_extend:DI
4696 (not:SI (xor:SI
4697 (SHIFT:SI
4698 (match_operand:SI 1 "register_operand" "r")
4699 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4700 (match_operand:SI 3 "register_operand" "r")))))]
4701 ""
4702 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4703 [(set_attr "type" "logic_shift_imm")]
4704)
4705
add3c965
IB		 4706(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4707 [(set (reg:CC_NZ CC_REGNUM)
4708 (compare:CC_NZ
4709 (and:GPI (not:GPI
4710 (SHIFT:GPI
4711 (match_operand:GPI 1 "register_operand" "r")
4712 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4713 (match_operand:GPI 3 "register_operand" "r"))
4714 (const_int 0)))
4715 (set (match_operand:GPI 0 "register_operand" "=r")
4716 (and:GPI (not:GPI
4717 (SHIFT:GPI
4718 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4719 ""
4720 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
99e6af15
JG		 4721 [(set_attr "type" "logics_shift_imm")]
4722)
add3c965
IB		 4723
4724;; zero_extend version of above
4725(define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4726 [(set (reg:CC_NZ CC_REGNUM)
4727 (compare:CC_NZ
4728 (and:SI (not:SI
4729 (SHIFT:SI
4730 (match_operand:SI 1 "register_operand" "r")
4731 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4732 (match_operand:SI 3 "register_operand" "r"))
4733 (const_int 0)))
4734 (set (match_operand:DI 0 "register_operand" "=r")
4735 (zero_extend:DI (and:SI
4736 (not:SI
4737 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4738 ""
4739 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
99e6af15
JG		 4740 [(set_attr "type" "logics_shift_imm")]
4741)
add3c965 4742
cd8fc5ee
AV		 4743(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4744 [(set (reg:CC_NZ CC_REGNUM)
4745 (compare:CC_NZ
4746 (and:GPI (not:GPI
4747 (SHIFT:GPI
4748 (match_operand:GPI 0 "register_operand" "r")
4749 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4750 (match_operand:GPI 2 "register_operand" "r"))
4751 (const_int 0)))]
4752 ""
4753 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4754 [(set_attr "type" "logics_shift_imm")]
4755)
4756
43e9d192
IB		 4757(define_insn "clz<mode>2"
4758 [(set (match_operand:GPI 0 "register_operand" "=r")
4759 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4760 ""
4761 "clz\\t%<w>0, %<w>1"
99e6af15
JG		 4762 [(set_attr "type" "clz")]
4763)
43e9d192
IB		 4764
4765(define_expand "ffs<mode>2"
4766 [(match_operand:GPI 0 "register_operand")
4767 (match_operand:GPI 1 "register_operand")]
4768 ""
4769 {
4770 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4771 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4772
4773 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4774 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
cd5660ab 4775 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
43e9d192
IB		 4776 DONE;
4777 }
4778)
4779
9d5019a1
AP		 4780;; Pop count be done via the "CNT" instruction in AdvSIMD.
4781;;
4782;; MOV v.1d, x0
4783;; CNT v1.8b, v.8b
4784;; ADDV b2, v1.8b
4785;; MOV w0, v2.b[0]
4786
4787(define_expand "popcount<mode>2"
4788 [(match_operand:GPI 0 "register_operand")
4789 (match_operand:GPI 1 "register_operand")]
4790 "TARGET_SIMD"
4791{
4792 rtx v = gen_reg_rtx (V8QImode);
4793 rtx v1 = gen_reg_rtx (V8QImode);
4794 rtx r = gen_reg_rtx (QImode);
4795 rtx in = operands[1];
4796 rtx out = operands[0];
4797 if(<MODE>mode == SImode)
4798 {
4799 rtx tmp;
4800 tmp = gen_reg_rtx (DImode);
4801 /* If we have SImode, zero extend to DImode, pop count does
4802 not change if we have extra zeros. */
4803 emit_insn (gen_zero_extendsidi2 (tmp, in));
4804 in = tmp;
4805 }
4806 emit_move_insn (v, gen_lowpart (V8QImode, in));
4807 emit_insn (gen_popcountv8qi2 (v1, v));
4808 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4809 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4810 DONE;
4811})
4812
43e9d192
IB		 4813(define_insn "clrsb<mode>2"
4814 [(set (match_operand:GPI 0 "register_operand" "=r")
dfc98d99 4815 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
43e9d192
IB		 4816 ""
4817 "cls\\t%<w>0, %<w>1"
99e6af15
JG		 4818 [(set_attr "type" "clz")]
4819)
43e9d192
IB		 4820
4821(define_insn "rbit<mode>2"
4822 [(set (match_operand:GPI 0 "register_operand" "=r")
4823 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4824 ""
4825 "rbit\\t%<w>0, %<w>1"
99e6af15
JG		 4826 [(set_attr "type" "rbit")]
4827)
43e9d192 4828
e8536e2b
KT		 4829;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4830;; it is unlikely to fold with any other operation, so keep this as a CTZ
4831;; expression and split after reload to enable scheduling them apart if
4832;; needed.
4833
4834(define_insn_and_split "ctz<mode>2"
4835 [(set (match_operand:GPI 0 "register_operand" "=r")
4836 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
43e9d192 4837 ""
e8536e2b
KT		 4838 "#"
4839 "reload_completed"
4840 [(const_int 0)]
4841 "
4842 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4843 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4844 DONE;
4845")
43e9d192 4846
2b8568fe
KT		 4847(define_insn "*and<mode>_compare0"
4848 [(set (reg:CC_NZ CC_REGNUM)
4849 (compare:CC_NZ
4850 (match_operand:SHORT 0 "register_operand" "r")
4851 (const_int 0)))]
4852 ""
4853 "tst\\t%<w>0, <short_mask>"
4854 [(set_attr "type" "alus_imm")]
4855)
4856
e7ba492a 4857(define_insn "*ands<GPI:mode>_compare0"
b06335f9
KT		 4858 [(set (reg:CC_NZ CC_REGNUM)
4859 (compare:CC_NZ
4860 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4861 (const_int 0)))
4862 (set (match_operand:GPI 0 "register_operand" "=r")
4863 (zero_extend:GPI (match_dup 1)))]
4864 ""
4865 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4866 [(set_attr "type" "alus_imm")]
4867)
4868
43e9d192 4869(define_insn "*and<mode>3nr_compare0"
d803a491
IB		 4870 [(set (reg:CC_NZ CC_REGNUM)
4871 (compare:CC_NZ
43e9d192
IB		 4872 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4873 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4874 (const_int 0)))]
4875 ""
954224d7
WD		 4876 "@
4877 tst\\t%<w>0, %<w>1
4878 tst\\t%<w>0, %1"
1d70b920 4879 [(set_attr "type" "logics_reg,logics_imm")]
99e6af15 4880)
43e9d192 4881
2c2789d5
KT		 4882(define_split
4883 [(set (reg:CC_NZ CC_REGNUM)
4884 (compare:CC_NZ
4885 (and:GPI (match_operand:GPI 0 "register_operand")
4886 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4887 (const_int 0)))
4888 (clobber (match_operand:SI 2 "register_operand"))]
4889 ""
4890 [(set (match_dup 2) (match_dup 1))
4891 (set (reg:CC_NZ CC_REGNUM)
4892 (compare:CC_NZ
4893 (and:GPI (match_dup 0)
4894 (match_dup 2))
4895 (const_int 0)))]
4896)
4897
7325d85a
KT		 4898(define_insn "*and<mode>3nr_compare0_zextract"
4899 [(set (reg:CC_NZ CC_REGNUM)
4900 (compare:CC_NZ
4901 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4902 (match_operand:GPI 1 "const_int_operand" "n")
4903 (match_operand:GPI 2 "const_int_operand" "n"))
4904 (const_int 0)))]
4905 "INTVAL (operands[1]) > 0
4906 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4907 <= GET_MODE_BITSIZE (<MODE>mode))
4908 && aarch64_bitmask_imm (
4909 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4910 operands[2])),
4911 <MODE>mode)"
4912 {
4913 operands[1]
4914 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4915 return "tst\\t%<w>0, %1";
4916 }
4917 [(set_attr "type" "logics_shift_imm")]
4918)
4919
43e9d192 4920(define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
d803a491
IB		 4921 [(set (reg:CC_NZ CC_REGNUM)
4922 (compare:CC_NZ
43e9d192
IB		 4923 (and:GPI (SHIFT:GPI
4924 (match_operand:GPI 0 "register_operand" "r")
4925 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4926 (match_operand:GPI 2 "register_operand" "r"))
4927 (const_int 0)))]
4928 ""
4929 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
99e6af15
JG		 4930 [(set_attr "type" "logics_shift_imm")]
4931)
43e9d192 4932
2c2789d5
KT		 4933(define_split
4934 [(set (reg:CC_NZ CC_REGNUM)
4935 (compare:CC_NZ
4936 (and:GPI (SHIFT:GPI
4937 (match_operand:GPI 0 "register_operand")
4938 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4939 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4940 (const_int 0)))
4941 (clobber (match_operand:SI 3 "register_operand"))]
4942 ""
4943 [(set (match_dup 3) (match_dup 2))
4944 (set (reg:CC_NZ CC_REGNUM)
4945 (compare:CC_NZ
4946 (and:GPI (SHIFT:GPI
4947 (match_dup 0)
4948 (match_dup 1))
4949 (match_dup 3))
4950 (const_int 0)))]
4951)
4952
43e9d192
IB		 4953;; -------------------------------------------------------------------
4954;; Shifts
4955;; -------------------------------------------------------------------
4956
4957(define_expand "<optab><mode>3"
4958 [(set (match_operand:GPI 0 "register_operand")
4959 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 4960 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 4961 ""
4962 {
4963 if (CONST_INT_P (operands[2]))
4964 {
4965 operands[2] = GEN_INT (INTVAL (operands[2])
4966 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4967
4968 if (operands[2] == const0_rtx)
4969 {
4970 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4971 DONE;
4972 }
4973 }
4974 }
4975)
4976
4977(define_expand "ashl<mode>3"
4978 [(set (match_operand:SHORT 0 "register_operand")
4979 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4bc72f35 4980 (match_operand:QI 2 "const_int_operand")))]
43e9d192
IB		 4981 ""
4982 {
4bc72f35 4983 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
43e9d192 4984
4bc72f35
KT		 4985 if (operands[2] == const0_rtx)
4986 {
4987 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4988 DONE;
43e9d192
IB		 4989 }
4990 }
4991)
4992
4993(define_expand "rotr<mode>3"
4994 [(set (match_operand:GPI 0 "register_operand")
4995 (rotatert:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 4996 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 4997 ""
4998 {
4999 if (CONST_INT_P (operands[2]))
5000 {
5001 operands[2] = GEN_INT (INTVAL (operands[2])
5002 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5003
5004 if (operands[2] == const0_rtx)
5005 {
5006 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5007 DONE;
5008 }
5009 }
5010 }
5011)
5012
5013(define_expand "rotl<mode>3"
5014 [(set (match_operand:GPI 0 "register_operand")
5015 (rotatert:GPI (match_operand:GPI 1 "register_operand")
f32c3adb 5016 (match_operand:QI 2 "aarch64_reg_or_imm")))]
43e9d192
IB		 5017 ""
5018 {
5019 /* (SZ - cnt) % SZ == -cnt % SZ */
5020 if (CONST_INT_P (operands[2]))
5021 {
5022 operands[2] = GEN_INT ((-INTVAL (operands[2]))
5023 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5024 if (operands[2] == const0_rtx)
5025 {
5026 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5027 DONE;
5028 }
5029 }
5030 else
5031 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
5032 NULL_RTX, 1);
5033 }
5034)
5035
7813b280
KT		 5036;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5037;; they truncate the shift/rotate amount by the size of the registers they
5038;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
5039;; such redundant masking instructions. GCC can do that automatically when
5040;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5041;; because some of the SISD shift alternatives don't perform this truncations.
5042;; So this pattern exists to catch such cases.
5043
5044(define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5045 [(set (match_operand:GPI 0 "register_operand" "=r")
5046 (SHIFT:GPI
5047 (match_operand:GPI 1 "register_operand" "r")
5048 (match_operator 4 "subreg_lowpart_operator"
5049 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5050 (match_operand 3 "const_int_operand" "n"))])))]
5051 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5052 "<shift>\t%<w>0, %<w>1, %<w>2"
5053 [(set_attr "type" "shift_reg")]
5054)
5055
cdeba3e0 5056(define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
7813b280
KT		 5057 [(set (match_operand:GPI 0 "register_operand" "=&r")
5058 (SHIFT:GPI
5059 (match_operand:GPI 1 "register_operand" "r")
5060 (match_operator 4 "subreg_lowpart_operator"
5061 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5062 (match_operand 3 "const_int_operand" "n")))])))]
5063 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5064 "#"
5065 "&& true"
5066 [(const_int 0)]
5067 {
5068 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
cdeba3e0 5069 : lowpart_subreg (SImode, operands[0], <MODE>mode));
7813b280
KT		 5070 emit_insn (gen_negsi2 (tmp, operands[2]));
5071
5072 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5073 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5074 SUBREG_BYTE (operands[4]));
5075 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5076 DONE;
5077 }
5078)
5079
cdeba3e0 5080(define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
7813b280
KT		 5081 [(set (match_operand:GPI 0 "register_operand" "=&r")
5082 (ashift:GPI
5083 (match_operand:GPI 1 "register_operand" "r")
5084 (minus:QI (match_operand 2 "const_int_operand" "n")
5085 (match_operator 5 "subreg_lowpart_operator"
5086 [(and:SI (match_operand:SI 3 "register_operand" "r")
5087 (match_operand 4 "const_int_operand" "n"))]))))]
5088 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5089 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5090 "#"
5091 "&& true"
5092 [(const_int 0)]
5093 {
5094 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5095 : operands[0]);
5096
5097 emit_insn (gen_negsi2 (tmp, operands[3]));
5098
5099 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5100 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5101 SUBREG_BYTE (operands[5]));
5102
5103 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5104 DONE;
5105 }
5106)
5107
5108(define_insn "*aarch64_<optab>_reg_di3_mask2"
5109 [(set (match_operand:DI 0 "register_operand" "=r")
5110 (SHIFT:DI
5111 (match_operand:DI 1 "register_operand" "r")
5112 (match_operator 4 "subreg_lowpart_operator"
5113 [(and:SI (match_operand:SI 2 "register_operand" "r")
cdeba3e0
JJ		 5114 (match_operand 3 "const_int_operand" "n"))])))]
5115 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
7813b280
KT		 5116{
5117 rtx xop[3];
5118 xop[0] = operands[0];
5119 xop[1] = operands[1];
5120 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5121 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5122 return "";
5123}
5124 [(set_attr "type" "shift_reg")]
5125)
5126
cdeba3e0 5127(define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
59abe903
MC		 5128 [(set (match_operand:GPI 0 "register_operand" "=&r")
5129 (ASHIFT:GPI
5130 (match_operand:GPI 1 "register_operand" "r")
5131 (minus:QI (match_operand 2 "const_int_operand" "n")
5132 (match_operand:QI 3 "register_operand" "r"))))]
5133 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5134 "#"
5135 "&& true"
5136 [(const_int 0)]
5137 {
5138 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5139
5140 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5141 : gen_lowpart (SImode, operands[0]));
5142
5143 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5144
5145 rtx and_op = gen_rtx_AND (SImode, tmp,
5146 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5147
5148 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5149
5150 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5151 DONE;
5152 }
5153 [(set_attr "length" "8")]
5154)
5155
2e100703
VP		 5156;; Logical left shift using SISD or Integer instruction
5157(define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
3362d764
WD		 5158 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5159 (ashift:GPI
5160 (match_operand:GPI 1 "register_operand" "r,r,w,w")
5161 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
2e100703
VP		 5162 ""
5163 "@
3362d764 5164 lsl\t%<w>0, %<w>1, %2
d99dacc9 5165 lsl\t%<w>0, %<w>1, %<w>2
2e100703 5166 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
d99dacc9 5167 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
488461d8
MM		 5168 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5169 (set_attr "arch" "*,*,simd,simd")]
2e100703
VP		 5170)
5171
5172;; Logical right shift using SISD or Integer instruction
5173(define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
3362d764
WD		 5174 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5175 (lshiftrt:GPI
5176 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
0603375c
KT		 5177 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5178 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
2e100703
VP		 5179 ""
5180 "@
3362d764 5181 lsr\t%<w>0, %<w>1, %2
ad7b853a 5182 lsr\t%<w>0, %<w>1, %<w>2
2e100703
VP		 5183 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5184 #
ad7b853a 5185 #"
488461d8
MM		 5186 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5187 (set_attr "arch" "*,*,simd,simd,simd")]
2e100703
VP		 5188)
5189
5190(define_split
5191 [(set (match_operand:DI 0 "aarch64_simd_register")
5192 (lshiftrt:DI
5193 (match_operand:DI 1 "aarch64_simd_register")
5194 (match_operand:QI 2 "aarch64_simd_register")))]
5195 "TARGET_SIMD && reload_completed"
498b13e2 5196 [(set (match_dup 3)
2e100703
VP		 5197 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5198 (set (match_dup 0)
498b13e2
KT		 5199 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5200 {
5201 operands[3] = gen_lowpart (QImode, operands[0]);
5202 }
2e100703
VP		 5203)
5204
5205(define_split
5206 [(set (match_operand:SI 0 "aarch64_simd_register")
5207 (lshiftrt:SI
5208 (match_operand:SI 1 "aarch64_simd_register")
5209 (match_operand:QI 2 "aarch64_simd_register")))]
5210 "TARGET_SIMD && reload_completed"
498b13e2 5211 [(set (match_dup 3)
2e100703
VP		 5212 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5213 (set (match_dup 0)
498b13e2
KT		 5214 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5215 {
5216 operands[3] = gen_lowpart (QImode, operands[0]);
5217 }
2e100703
VP		 5218)
5219
5220;; Arithmetic right shift using SISD or Integer instruction
5221(define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
3362d764 5222 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
0603375c
KT		 5223 (ashiftrt:GPI
5224 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5225 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5226 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
2e100703
VP		 5227 ""
5228 "@
3362d764 5229 asr\t%<w>0, %<w>1, %2
ad7b853a 5230 asr\t%<w>0, %<w>1, %<w>2
2e100703
VP		 5231 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5232 #
ad7b853a 5233 #"
488461d8
MM		 5234 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5235 (set_attr "arch" "*,*,simd,simd,simd")]
2e100703
VP		 5236)
5237
5238(define_split
5239 [(set (match_operand:DI 0 "aarch64_simd_register")
5240 (ashiftrt:DI
5241 (match_operand:DI 1 "aarch64_simd_register")
5242 (match_operand:QI 2 "aarch64_simd_register")))]
5243 "TARGET_SIMD && reload_completed"
21a7b5e1 5244 [(set (match_dup 3)
2e100703
VP		 5245 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5246 (set (match_dup 0)
21a7b5e1
MS		 5247 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5248{
5249 operands[3] = gen_lowpart (QImode, operands[0]);
5250}
2e100703
VP		 5251)
5252
5253(define_split
5254 [(set (match_operand:SI 0 "aarch64_simd_register")
5255 (ashiftrt:SI
5256 (match_operand:SI 1 "aarch64_simd_register")
5257 (match_operand:QI 2 "aarch64_simd_register")))]
5258 "TARGET_SIMD && reload_completed"
21a7b5e1 5259 [(set (match_dup 3)
2e100703
VP		 5260 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5261 (set (match_dup 0)
21a7b5e1
MS		 5262 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5263{
5264 operands[3] = gen_lowpart (QImode, operands[0]);
5265}
2e100703
VP		 5266)
5267
5268(define_insn "*aarch64_sisd_ushl"
5269 [(set (match_operand:DI 0 "register_operand" "=w")
5270 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5271 (match_operand:QI 2 "register_operand" "w")]
5272 UNSPEC_SISD_USHL))]
5273 "TARGET_SIMD"
5274 "ushl\t%d0, %d1, %d2"
488461d8 5275 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5276)
5277
5278(define_insn "*aarch64_ushl_2s"
5279 [(set (match_operand:SI 0 "register_operand" "=w")
5280 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5281 (match_operand:QI 2 "register_operand" "w")]
5282 UNSPEC_USHL_2S))]
5283 "TARGET_SIMD"
5284 "ushl\t%0.2s, %1.2s, %2.2s"
488461d8 5285 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5286)
5287
5288(define_insn "*aarch64_sisd_sshl"
5289 [(set (match_operand:DI 0 "register_operand" "=w")
5290 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5291 (match_operand:QI 2 "register_operand" "w")]
5292 UNSPEC_SISD_SSHL))]
5293 "TARGET_SIMD"
5294 "sshl\t%d0, %d1, %d2"
488461d8 5295 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5296)
5297
5298(define_insn "*aarch64_sshl_2s"
5299 [(set (match_operand:SI 0 "register_operand" "=w")
5300 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5301 (match_operand:QI 2 "register_operand" "w")]
5302 UNSPEC_SSHL_2S))]
5303 "TARGET_SIMD"
5304 "sshl\t%0.2s, %1.2s, %2.2s"
488461d8 5305 [(set_attr "type" "neon_shift_reg")]
2e100703
VP		 5306)
5307
5308(define_insn "*aarch64_sisd_neg_qi"
5309 [(set (match_operand:QI 0 "register_operand" "=w")
5310 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5311 UNSPEC_SISD_NEG))]
5312 "TARGET_SIMD"
5313 "neg\t%d0, %d1"
488461d8 5314 [(set_attr "type" "neon_neg")]
2e100703
VP		 5315)
5316
5317;; Rotate right
5318(define_insn "*ror<mode>3_insn"
b81f1ee3
JG		 5319 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5320 (rotatert:GPI
5321 (match_operand:GPI 1 "register_operand" "r,r")
3362d764 5322 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
43e9d192 5323 ""
3362d764
WD		 5324 "@
5325 ror\\t%<w>0, %<w>1, %2
5326 ror\\t%<w>0, %<w>1, %<w>2"
5327 [(set_attr "type" "rotate_imm,shift_reg")]
43e9d192
IB		 5328)
5329
d0b6bb1b
IB		 5330;; zero_extend version of above
5331(define_insn "*<optab>si3_insn_uxtw"
3362d764 5332 [(set (match_operand:DI 0 "register_operand" "=r,r")
d0b6bb1b 5333 (zero_extend:DI (SHIFT:SI
3362d764
WD		 5334 (match_operand:SI 1 "register_operand" "r,r")
5335 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
d0b6bb1b 5336 ""
3362d764
WD		 5337 "@
5338 <shift>\\t%w0, %w1, %2
5339 <shift>\\t%w0, %w1, %w2"
94f7a25e 5340 [(set_attr "type" "bfx,shift_reg")]
d0b6bb1b
IB		 5341)
5342
43e9d192
IB		 5343(define_insn "*<optab><mode>3_insn"
5344 [(set (match_operand:SHORT 0 "register_operand" "=r")
5345 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5346 (match_operand 2 "const_int_operand" "n")))]
5347 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5348{
5349 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5350 return "<bfshift>\t%w0, %w1, %2, %3";
5351}
94f7a25e 5352 [(set_attr "type" "bfx")]
43e9d192
IB		 5353)
5354
bd83ff2c
IB		 5355(define_insn "*extr<mode>5_insn"
5356 [(set (match_operand:GPI 0 "register_operand" "=r")
5357 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5358 (match_operand 3 "const_int_operand" "n"))
5359 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5360 (match_operand 4 "const_int_operand" "n"))))]
5361 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5362 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5363 "extr\\t%<w>0, %<w>1, %<w>2, %4"
3362d764 5364 [(set_attr "type" "rotate_imm")]
bd83ff2c
IB		 5365)
5366
fb0cb7fa
KT		 5367;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5368;; so we have to match both orderings.
5369(define_insn "*extr<mode>5_insn_alt"
5370 [(set (match_operand:GPI 0 "register_operand" "=r")
5371 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5372 (match_operand 4 "const_int_operand" "n"))
5373 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5374 (match_operand 3 "const_int_operand" "n"))))]
5375 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5376 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5377 == GET_MODE_BITSIZE (<MODE>mode))"
5378 "extr\\t%<w>0, %<w>1, %<w>2, %4"
3362d764 5379 [(set_attr "type" "rotate_imm")]
fb0cb7fa
KT		 5380)
5381
bd83ff2c
IB		 5382;; zero_extend version of the above
5383(define_insn "*extrsi5_insn_uxtw"
5384 [(set (match_operand:DI 0 "register_operand" "=r")
5385 (zero_extend:DI
5386 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5387 (match_operand 3 "const_int_operand" "n"))
5388 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5389 (match_operand 4 "const_int_operand" "n")))))]
5390 "UINTVAL (operands[3]) < 32 &&
5391 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5392 "extr\\t%w0, %w1, %w2, %4"
3362d764 5393 [(set_attr "type" "rotate_imm")]
bd83ff2c
IB		 5394)
5395
fb0cb7fa
KT		 5396(define_insn "*extrsi5_insn_uxtw_alt"
5397 [(set (match_operand:DI 0 "register_operand" "=r")
5398 (zero_extend:DI
5399 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5400 (match_operand 4 "const_int_operand" "n"))
5401 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5402 (match_operand 3 "const_int_operand" "n")))))]
5403 "UINTVAL (operands[3]) < 32 &&
5404 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5405 "extr\\t%w0, %w1, %w2, %4"
3362d764 5406 [(set_attr "type" "rotate_imm")]
fb0cb7fa
KT		 5407)
5408
5977a10d
IB		 5409(define_insn "*ror<mode>3_insn"
5410 [(set (match_operand:GPI 0 "register_operand" "=r")
5411 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5412 (match_operand 2 "const_int_operand" "n")))]
5413 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5414{
5415 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5416 return "ror\\t%<w>0, %<w>1, %3";
5417}
b81f1ee3 5418 [(set_attr "type" "rotate_imm")]
5977a10d
IB		 5419)
5420
5421;; zero_extend version of the above
5422(define_insn "*rorsi3_insn_uxtw"
5423 [(set (match_operand:DI 0 "register_operand" "=r")
5424 (zero_extend:DI
5425 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5426 (match_operand 2 "const_int_operand" "n"))))]
5427 "UINTVAL (operands[2]) < 32"
5428{
5429 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5430 return "ror\\t%w0, %w1, %3";
5431}
b81f1ee3 5432 [(set_attr "type" "rotate_imm")]
5977a10d
IB		 5433)
5434
43e9d192
IB		 5435(define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5436 [(set (match_operand:GPI 0 "register_operand" "=r")
5437 (ANY_EXTEND:GPI
5438 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5439 (match_operand 2 "const_int_operand" "n"))))]
5440 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5441{
5442 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5443 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5444}
94f7a25e 5445 [(set_attr "type" "bfx")]
43e9d192
IB		 5446)
5447
5448(define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5449 [(set (match_operand:GPI 0 "register_operand" "=r")
5450 (zero_extend:GPI
5451 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5452 (match_operand 2 "const_int_operand" "n"))))]
5453 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5454{
5455 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5456 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5457}
94f7a25e 5458 [(set_attr "type" "bfx")]
43e9d192
IB		 5459)
5460
5461(define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5462 [(set (match_operand:GPI 0 "register_operand" "=r")
5463 (sign_extend:GPI
5464 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5465 (match_operand 2 "const_int_operand" "n"))))]
5466 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5467{
5468 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5469 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5470}
94f7a25e 5471 [(set_attr "type" "bfx")]
43e9d192
IB		 5472)
5473
5474;; -------------------------------------------------------------------
5475;; Bitfields
5476;; -------------------------------------------------------------------
5477
5478(define_expand "<optab>"
1bbffb87 5479 [(set (match_operand:DI 0 "register_operand")
e27bfda2
KT		 5480 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5481 (match_operand 2
5482 "aarch64_simd_shift_imm_offset_di")
5483 (match_operand 3 "aarch64_simd_shift_imm_di")))]
43e9d192 5484 ""
e27bfda2
KT		 5485 {
5486 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5487 1, GET_MODE_BITSIZE (DImode) - 1))
5488 FAIL;
5489 }
43e9d192
IB		 5490)
5491
e27bfda2 5492
43e9d192
IB		 5493(define_insn "*<optab><mode>"
5494 [(set (match_operand:GPI 0 "register_operand" "=r")
5495 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
e27bfda2
KT		 5496 (match_operand 2
5497 "aarch64_simd_shift_imm_offset_<mode>" "n")
5498 (match_operand 3
5499 "aarch64_simd_shift_imm_<mode>" "n")))]
5500 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5501 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
43e9d192 5502 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
94f7a25e 5503 [(set_attr "type" "bfx")]
65f9e789
IB		 5504)
5505
bcb036c5
KT		 5506;; When the bit position and width add up to 32 we can use a W-reg LSR
5507;; instruction taking advantage of the implicit zero-extension of the X-reg.
5508(define_split
5509 [(set (match_operand:DI 0 "register_operand")
5510 (zero_extract:DI (match_operand:DI 1 "register_operand")
5511 (match_operand 2
5512 "aarch64_simd_shift_imm_offset_di")
5513 (match_operand 3
5514 "aarch64_simd_shift_imm_di")))]
5515 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5516 GET_MODE_BITSIZE (DImode) - 1)
5517 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5518 == GET_MODE_BITSIZE (SImode)"
5519 [(set (match_dup 0)
5520 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5521 {
5522 operands[4] = gen_lowpart (SImode, operands[1]);
5523 }
5524)
5525
65f9e789
IB		 5526;; Bitfield Insert (insv)
5527(define_expand "insv<mode>"
5528 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5529 (match_operand 1 "const_int_operand")
5530 (match_operand 2 "const_int_operand"))
5531 (match_operand:GPI 3 "general_operand"))]
5532 ""
5533{
5534 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5535 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5536 rtx value = operands[3];
5537
5538 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5539 FAIL;
5540
5541 if (CONST_INT_P (value))
5542 {
5543 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5544
5545 /* Prefer AND/OR for inserting all zeros or all ones. */
5546 if ((UINTVAL (value) & mask) == 0
5547 || (UINTVAL (value) & mask) == mask)
5548 FAIL;
5549
5550 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5551 if (width == 16 && (pos % 16) == 0)
5552 DONE;
5553 }
5554 operands[3] = force_reg (<MODE>mode, value);
5555})
5556
5557(define_insn "*insv_reg<mode>"
5558 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5559 (match_operand 1 "const_int_operand" "n")
5560 (match_operand 2 "const_int_operand" "n"))
5561 (match_operand:GPI 3 "register_operand" "r"))]
5562 "!(UINTVAL (operands[1]) == 0
5563 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5564 > GET_MODE_BITSIZE (<MODE>mode)))"
5565 "bfi\\t%<w>0, %<w>3, %2, %1"
99e6af15 5566 [(set_attr "type" "bfm")]
2879bb2b
IB		 5567)
5568
e4f019d3
RL		 5569(define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5570 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5571 (match_operand 1 "const_int_operand" "n")
5572 (match_operand 2 "const_int_operand" "n"))
5573 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5574 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5575 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5576 [(set_attr "type" "bfm")]
5577)
5578
6a0d3939
SE		 5579;; Match a bfi instruction where the shift of OP3 means that we are
5580;; actually copying the least significant bits of OP3 into OP0 by way
5581;; of the AND masks and the IOR instruction. A similar instruction
5582;; with the two parts of the IOR swapped around was never triggered
5583;; in a bootstrap build and test of GCC so it was not included.
5584
5585(define_insn "*aarch64_bfi<GPI:mode>5_shift"
5586 [(set (match_operand:GPI 0 "register_operand" "=r")
5587 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5588 (match_operand:GPI 2 "const_int_operand" "n"))
5589 (and:GPI (ashift:GPI
5590 (match_operand:GPI 3 "register_operand" "r")
5591 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5592 (match_operand:GPI 5 "const_int_operand" "n"))))]
5593 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5594 UINTVAL (operands[4]),
5595 UINTVAL(operands[5]))"
5596 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5597 [(set_attr "type" "bfm")]
5598)
5599
5600(define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5601 [(set (match_operand:GPI 0 "register_operand" "=r")
5602 (ior:GPI (and:GPI (ashift:GPI
5603 (match_operand:GPI 1 "register_operand" "r")
5604 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5605 (match_operand:GPI 3 "const_int_operand" "n"))
5606 (and:GPI (match_operand:GPI 4 "register_operand" "0")
5607 (match_operand:GPI 5 "const_int_operand" "n"))))]
5608 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5609 UINTVAL (operands[2]),
5610 UINTVAL(operands[3]))"
5611 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5612 [(set_attr "type" "bfm")]
5613)
5614
5615;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5616;; the shift is large enough to remove the need for an AND instruction.
5617
5618(define_insn "*aarch64_bfi<GPI:mode>4_noand"
5619 [(set (match_operand:GPI 0 "register_operand" "=r")
5620 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5621 (match_operand:GPI 2 "const_int_operand" "n"))
5622 (ashift:GPI
5623 (match_operand:GPI 3 "register_operand" "r")
5624 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5625 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5626 UINTVAL (operands[4]),
5627 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5628{
5629 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5630 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5631}
5632 [(set_attr "type" "bfm")]
5633)
5634
5635(define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5636 [(set (match_operand:GPI 0 "register_operand" "=r")
5637 (ior:GPI (ashift:GPI
5638 (match_operand:GPI 1 "register_operand" "r")
5639 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5640 (and:GPI (match_operand:GPI 3 "register_operand" "0")
5641 (match_operand:GPI 4 "const_int_operand" "n"))))]
5642 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5643 UINTVAL (operands[2]),
5644 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5645{
5646 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5647 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5648}
5649 [(set_attr "type" "bfm")]
5650)
5651
5652;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
f0a8f6bc
SE		 5653;; copying the least significant bits of OP3 to OP0. We need two versions
5654;; of the instruction to handle different checks on the constant values.
6a0d3939
SE		 5655
5656(define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5657 [(set (match_operand:GPI 0 "register_operand" "=r")
5658 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5659 (match_operand:GPI 2 "const_int_operand" "n"))
5660 (and:GPI (match_operand:GPI 3 "register_operand" "r")
5661 (match_operand:GPI 4 "const_int_operand" "n"))))]
5662 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5663 UINTVAL (operands[4]))"
5664 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5665 [(set_attr "type" "bfm")]
5666)
5667
f0a8f6bc
SE		 5668(define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5669 [(set (match_operand:GPI 0 "register_operand" "=r")
5670 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5671 (match_operand:GPI 4 "const_int_operand" "n"))
5672 (and:GPI (match_operand:GPI 1 "register_operand" "0")
5673 (match_operand:GPI 2 "const_int_operand" "n"))))]
5674 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5675 UINTVAL (operands[4]))"
5676 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5677 [(set_attr "type" "bfm")]
5678)
5679
2879bb2b
IB		 5680(define_insn "*extr_insv_lower_reg<mode>"
5681 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5682 (match_operand 1 "const_int_operand" "n")
5683 (const_int 0))
98fa5ef3 5684 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
2879bb2b
IB		 5685 (match_dup 1)
5686 (match_operand 3 "const_int_operand" "n")))]
5687 "!(UINTVAL (operands[1]) == 0
5688 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5689 > GET_MODE_BITSIZE (<MODE>mode)))"
5690 "bfxil\\t%<w>0, %<w>2, %3, %1"
99e6af15 5691 [(set_attr "type" "bfm")]
43e9d192
IB		 5692)
5693
5694(define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5695 [(set (match_operand:GPI 0 "register_operand" "=r")
5696 (ashift:GPI (ANY_EXTEND:GPI
5697 (match_operand:ALLX 1 "register_operand" "r"))
5698 (match_operand 2 "const_int_operand" "n")))]
5699 "UINTVAL (operands[2]) < <GPI:sizen>"
5700{
5701 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5702 ? GEN_INT (<ALLX:sizen>)
5703 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5704 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5705}
94f7a25e 5706 [(set_attr "type" "bfx")]
43e9d192
IB		 5707)
5708
5709;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5710
5711(define_insn "*andim_ashift<mode>_bfiz"
5712 [(set (match_operand:GPI 0 "register_operand" "=r")
5713 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5714 (match_operand 2 "const_int_operand" "n"))
5715 (match_operand 3 "const_int_operand" "n")))]
8c83f71d 5716 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
43e9d192 5717 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
94f7a25e 5718 [(set_attr "type" "bfx")]
43e9d192
IB		 5719)
5720
58c2ad42
LM		 5721;; Match sbfiz pattern in a shift left + shift right operation.
5722
5723(define_insn "*ashift<mode>_extv_bfiz"
5724 [(set (match_operand:GPI 0 "register_operand" "=r")
5725 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5726 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5727 (const_int 0))
5728 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5729 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5730 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5731 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5732 [(set_attr "type" "bfx")]
5733)
5734
680153bd
KT		 5735;; When the bit position and width of the equivalent extraction add up to 32
5736;; we can use a W-reg LSL instruction taking advantage of the implicit
5737;; zero-extension of the X-reg.
5738(define_split
5739 [(set (match_operand:DI 0 "register_operand")
5740 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5741 (match_operand 2 "const_int_operand"))
5742 (match_operand 3 "const_int_operand")))]
5743 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5744 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5745 == GET_MODE_BITSIZE (SImode)"
5746 [(set (match_dup 0)
5747 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5748 {
5749 operands[4] = gen_lowpart (SImode, operands[1]);
5750 }
5751)
5752
43e9d192
IB		 5753(define_insn "bswap<mode>2"
5754 [(set (match_operand:GPI 0 "register_operand" "=r")
5755 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5756 ""
5757 "rev\\t%<w>0, %<w>1"
99e6af15 5758 [(set_attr "type" "rev")]
43e9d192
IB		 5759)
5760
c16c63bb
IB		 5761(define_insn "bswaphi2"
5762 [(set (match_operand:HI 0 "register_operand" "=r")
5763 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5764 ""
5765 "rev16\\t%w0, %w1"
99e6af15 5766 [(set_attr "type" "rev")]
c16c63bb
IB		 5767)
5768
231c52ae
ST		 5769(define_insn "*aarch64_bfxil<mode>"
5770 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5771 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5772 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5773 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5774 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5775 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5776 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5777 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5778 {
5779 switch (which_alternative)
5780 {
5781 case 0:
5782 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5783 return "bfxil\\t%<w>0, %<w>1, 0, %3";
5784 case 1:
5785 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5786 return "bfxil\\t%<w>0, %<w>2, 0, %3";
5787 default:
5788 gcc_unreachable ();
5789 }
5790 }
5791 [(set_attr "type" "bfm")]
5792)
5793
2dc09f66
ST		 5794; Zero-extended version of above (aarch64_bfxil)
5795(define_insn "*aarch64_bfxilsi_uxtw"
5796 [(set (match_operand:DI 0 "register_operand" "=r,r")
5797 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5798 "r,0")
5799 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5800 (and:SI (match_operand:SI 2 "register_operand" "0,r")
5801 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5802 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5803 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5804 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5805 {
5806 switch (which_alternative)
5807 {
5808 case 0:
5809 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5810 return "bfxil\\t%0, %1, 0, %3";
5811 case 1:
5812 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5813 return "bfxil\\t%0, %2, 0, %3";
5814 default:
5815 gcc_unreachable ();
5816 }
5817 }
5818 [(set_attr "type" "bfm")]
5819)
5820
f7d5cf8d
KT		 5821;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5822;; operations within an IOR/AND RTX, therefore we have two patterns matching
5823;; each valid permutation.
5824
5825(define_insn "rev16<mode>2"
5826 [(set (match_operand:GPI 0 "register_operand" "=r")
5827 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5828 (const_int 8))
5829 (match_operand:GPI 3 "const_int_operand" "n"))
5830 (and:GPI (lshiftrt:GPI (match_dup 1)
5831 (const_int 8))
5832 (match_operand:GPI 2 "const_int_operand" "n"))))]
5833 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5834 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5835 "rev16\\t%<w>0, %<w>1"
5836 [(set_attr "type" "rev")]
5837)
5838
5839(define_insn "rev16<mode>2_alt"
5840 [(set (match_operand:GPI 0 "register_operand" "=r")
5841 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5842 (const_int 8))
5843 (match_operand:GPI 2 "const_int_operand" "n"))
5844 (and:GPI (ashift:GPI (match_dup 1)
5845 (const_int 8))
5846 (match_operand:GPI 3 "const_int_operand" "n"))))]
5847 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5848 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5849 "rev16\\t%<w>0, %<w>1"
5850 [(set_attr "type" "rev")]
5851)
5852
d0b6bb1b
IB		 5853;; zero_extend version of above
5854(define_insn "*bswapsi2_uxtw"
5855 [(set (match_operand:DI 0 "register_operand" "=r")
5856 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5857 ""
5858 "rev\\t%w0, %w1"
99e6af15 5859 [(set_attr "type" "rev")]
d0b6bb1b
IB		 5860)
5861
43e9d192
IB		 5862;; -------------------------------------------------------------------
5863;; Floating-point intrinsics
5864;; -------------------------------------------------------------------
5865
42fc9a7f 5866;; frint floating-point round to integral standard patterns.
74dc11ed 5867;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
43e9d192 5868
42fc9a7f 5869(define_insn "<frint_pattern><mode>2"
d7f33f07
JW		 5870 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5871 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
42fc9a7f 5872 FRINT))]
43e9d192 5873 "TARGET_FLOAT"
42fc9a7f 5874 "frint<frint_suffix>\\t%<s>0, %<s>1"
d7f33f07 5875 [(set_attr "type" "f_rint<stype>")]
43e9d192
IB		 5876)
5877
42fc9a7f
JG		 5878;; frcvt floating-point round to integer and convert standard patterns.
5879;; Expands to lbtrunc, lceil, lfloor, lround.
d7f33f07 5880(define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
43e9d192 5881 [(set (match_operand:GPI 0 "register_operand" "=r")
d7f33f07
JW		 5882 (FIXUORS:GPI
5883 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5884 FCVT)))]
43e9d192 5885 "TARGET_FLOAT"
d7f33f07 5886 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
99e6af15 5887 [(set_attr "type" "f_cvtf2i")]
43e9d192
IB		 5888)
5889
39252973
KT		 5890(define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5891 [(set (match_operand:GPI 0 "register_operand" "=r")
5892 (FIXUORS:GPI
5893 (mult:GPF
5894 (match_operand:GPF 1 "register_operand" "w")
5895 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5896 "TARGET_FLOAT
5897 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5898 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5899 {
5900 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5901 char buf[64];
5902 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5903 output_asm_insn (buf, operands);
5904 return "";
5905 }
5906 [(set_attr "type" "f_cvtf2i")]
5907)
5908
d6e6e8b6
WD		 5909;; fma - expand fma into patterns with the accumulator operand first since
5910;; reusing the accumulator results in better register allocation.
5911;; The register allocator considers copy preferences in operand order,
5912;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5913
5914(define_expand "fma<mode>4"
5915 [(set (match_operand:GPF_F16 0 "register_operand")
5916 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5917 (match_operand:GPF_F16 2 "register_operand")
5918 (match_operand:GPF_F16 3 "register_operand")))]
5919 "TARGET_FLOAT"
5920)
43e9d192 5921
d6e6e8b6 5922(define_insn "*aarch64_fma<mode>4"
9a594ad6 5923 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
d6e6e8b6
WD		 5924 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5925 (match_operand:GPF_F16 3 "register_operand" "w")
5926 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192 5927 "TARGET_FLOAT"
d6e6e8b6 5928 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
9a594ad6 5929 [(set_attr "type" "fmac<stype>")]
43e9d192
IB		 5930)
5931
d6e6e8b6
WD		 5932(define_expand "fnma<mode>4"
5933 [(set (match_operand:GPF_F16 0 "register_operand")
5934 (fma:GPF_F16
5935 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5936 (match_operand:GPF_F16 2 "register_operand")
5937 (match_operand:GPF_F16 3 "register_operand")))]
5938 "TARGET_FLOAT"
5939)
5940
5941(define_insn "*aarch64_fnma<mode>4"
9a594ad6
JW		 5942 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5943 (fma:GPF_F16
d6e6e8b6
WD		 5944 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5945 (match_operand:GPF_F16 3 "register_operand" "w")
5946 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192 5947 "TARGET_FLOAT"
d6e6e8b6 5948 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
9a594ad6 5949 [(set_attr "type" "fmac<stype>")]
43e9d192
IB		 5950)
5951
d6e6e8b6
WD		 5952
5953(define_expand "fms<mode>4"
5954 [(set (match_operand:GPF 0 "register_operand")
5955 (fma:GPF (match_operand:GPF 1 "register_operand")
5956 (match_operand:GPF 2 "register_operand")
5957 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5958 "TARGET_FLOAT"
5959)
5960
5961(define_insn "*aarch64_fms<mode>4"
43e9d192 5962 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 5963 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5964 (match_operand:GPF 3 "register_operand" "w")
5965 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 5966 "TARGET_FLOAT"
d6e6e8b6 5967 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 5968 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 5969)
5970
d6e6e8b6
WD		 5971(define_expand "fnms<mode>4"
5972 [(set (match_operand:GPF 0 "register_operand")
5973 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5974 (match_operand:GPF 2 "register_operand")
5975 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5976 "TARGET_FLOAT"
5977)
5978
5979(define_insn "*aarch64_fnms<mode>4"
43e9d192 5980 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 5981 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5982 (match_operand:GPF 3 "register_operand" "w")
5983 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 5984 "TARGET_FLOAT"
d6e6e8b6 5985 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 5986 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 5987)
5988
5989;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
d6e6e8b6 5990(define_insn "*aarch64_fnmadd<mode>4"
43e9d192 5991 [(set (match_operand:GPF 0 "register_operand" "=w")
d6e6e8b6
WD		 5992 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5993 (match_operand:GPF 3 "register_operand" "w")
5994 (match_operand:GPF 1 "register_operand" "w"))))]
43e9d192 5995 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
d6e6e8b6 5996 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
99e6af15 5997 [(set_attr "type" "fmac<s>")]
43e9d192
IB		 5998)
5999
6000;; -------------------------------------------------------------------
6001;; Floating-point conversions
6002;; -------------------------------------------------------------------
6003
6004(define_insn "extendsfdf2"
6005 [(set (match_operand:DF 0 "register_operand" "=w")
6006 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
6007 "TARGET_FLOAT"
6008 "fcvt\\t%d0, %s1"
99e6af15 6009 [(set_attr "type" "f_cvt")]
43e9d192
IB		 6010)
6011
c2ec330c
AL		 6012(define_insn "extendhfsf2"
6013 [(set (match_operand:SF 0 "register_operand" "=w")
6014 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
6015 "TARGET_FLOAT"
6016 "fcvt\\t%s0, %h1"
6017 [(set_attr "type" "f_cvt")]
6018)
6019
6020(define_insn "extendhfdf2"
6021 [(set (match_operand:DF 0 "register_operand" "=w")
6022 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
6023 "TARGET_FLOAT"
6024 "fcvt\\t%d0, %h1"
6025 [(set_attr "type" "f_cvt")]
6026)
6027
43e9d192
IB		 6028(define_insn "truncdfsf2"
6029 [(set (match_operand:SF 0 "register_operand" "=w")
6030 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
6031 "TARGET_FLOAT"
6032 "fcvt\\t%s0, %d1"
99e6af15 6033 [(set_attr "type" "f_cvt")]
43e9d192
IB		 6034)
6035
c2ec330c
AL		 6036(define_insn "truncsfhf2"
6037 [(set (match_operand:HF 0 "register_operand" "=w")
6038 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6039 "TARGET_FLOAT"
6040 "fcvt\\t%h0, %s1"
6041 [(set_attr "type" "f_cvt")]
6042)
6043
6044(define_insn "truncdfhf2"
6045 [(set (match_operand:HF 0 "register_operand" "=w")
6046 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6047 "TARGET_FLOAT"
6048 "fcvt\\t%h0, %d1"
6049 [(set_attr "type" "f_cvt")]
6050)
6051
22be0d08
MC		 6052;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6053;; and making r = w more expensive
6054
6055(define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
43d0a8ee 6056 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
22be0d08
MC		 6057 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6058 "TARGET_FLOAT"
6059 "@
43d0a8ee
WD		 6060 fcvtz<su>\t%<s>0, %<s>1
6061 fcvtz<su>\t%<w>0, %<s>1"
6062 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
22be0d08
MC		 6063)
6064
6065;; Convert HF -> SI or DI
6066
6067(define_insn "<optab>_trunchf<GPI:mode>2"
6068 [(set (match_operand:GPI 0 "register_operand" "=r")
6069 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6070 "TARGET_FP_F16INST"
6071 "fcvtz<su>\t%<w>0, %h1"
6072 [(set_attr "type" "f_cvtf2i")]
6073)
6074
6075;; Convert DF -> SI or SF -> DI which can only be accomplished with
6076;; input in a fp register and output in a integer register
6077
6078(define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
43e9d192 6079 [(set (match_operand:GPI 0 "register_operand" "=r")
22be0d08 6080 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
43e9d192 6081 "TARGET_FLOAT"
22be0d08 6082 "fcvtz<su>\t%<w>0, %<fpw>1"
99e6af15 6083 [(set_attr "type" "f_cvtf2i")]
43e9d192
IB		 6084)
6085
eeb59c16
MC		 6086(define_insn "*fix_to_zero_extend<mode>di2"
6087 [(set (match_operand:DI 0 "register_operand" "=r")
6088 (zero_extend:DI
6089 (unsigned_fix:SI
6090 (match_operand:GPF 1 "register_operand" "w"))))]
6091 "TARGET_FLOAT"
6092 "fcvtzu\t%w0, %<s>1"
6093 [(set_attr "type" "f_cvtf2i")]
6094)
6095
188d0079
JH		 6096;; Equal width integer to fp and multiply combine.
6097(define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6098 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6099 (mult:GPF (FLOATUORS:GPF
6100 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6101 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6102 "TARGET_FLOAT"
6103 {
6104 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6105 switch (which_alternative)
6106 {
6107 case 0:
6108 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6109 case 1:
6110 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6111 default:
6112 gcc_unreachable ();
6113 }
6114 }
6115 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6116 (set_attr "arch" "simd,fp")]
6117)
6118
6119;; Unequal width integer to fp and multiply combine.
6120(define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6121 [(set (match_operand:GPF 0 "register_operand" "=w")
6122 (mult:GPF (FLOATUORS:GPF
6123 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6124 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6125 "TARGET_FLOAT"
6126 {
6127 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6128 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6129 }
6130 [(set_attr "type" "f_cvti2f")]
6131)
6132
6133;; Equal width integer to fp conversion.
0d35c5c2
VP		 6134(define_insn "<optab><fcvt_target><GPF:mode>2"
6135 [(set (match_operand:GPF 0 "register_operand" "=w,w")
43d0a8ee 6136 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
31ca7cba 6137 "TARGET_FLOAT"
0d35c5c2
VP		 6138 "@
6139 <su_optab>cvtf\t%<GPF:s>0, %<s>1
6140 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
488461d8
MM		 6141 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6142 (set_attr "arch" "simd,fp")]
43e9d192
IB		 6143)
6144
188d0079 6145;; Unequal width integer to fp conversions.
0d35c5c2 6146(define_insn "<optab><fcvt_iesize><GPF:mode>2"
43e9d192 6147 [(set (match_operand:GPF 0 "register_operand" "=w")
0d35c5c2 6148 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
43e9d192 6149 "TARGET_FLOAT"
0d35c5c2
VP		 6150 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6151 [(set_attr "type" "f_cvti2f")]
43e9d192
IB		 6152)
6153
bf1e3646
JG		 6154;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6155;; midend will arrange for an SImode conversion to HFmode to first go
6156;; through DFmode, then to HFmode. But first it will try converting
6157;; to DImode then down, which would match our DImode pattern below and
6158;; give very poor code-generation. So, we must provide our own emulation
6159;; of the mid-end logic.
6160
6161(define_insn "aarch64_fp16_<optab><mode>hf2"
d7f33f07
JW		 6162 [(set (match_operand:HF 0 "register_operand" "=w")
6163 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6164 "TARGET_FP_F16INST"
6165 "<su_optab>cvtf\t%h0, %<w>1"
6166 [(set_attr "type" "f_cvti2f")]
6167)
6168
bf1e3646
JG		 6169(define_expand "<optab>sihf2"
6170 [(set (match_operand:HF 0 "register_operand")
6171 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6172 "TARGET_FLOAT"
6173{
6174 if (TARGET_FP_F16INST)
6175 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6176 else
6177 {
6178 rtx convert_target = gen_reg_rtx (DFmode);
6179 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6180 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6181 }
6182 DONE;
6183}
6184)
6185
6186;; For DImode there is no wide enough floating-point mode that we
6187;; can convert through natively (TFmode would work, but requires a library
6188;; call). However, we know that any value >= 65504 will be rounded
6189;; to infinity on conversion. This is well within the range of SImode, so
6190;; we can:
6191;; Saturate to SImode.
6192;; Convert from that to DFmode
6193;; Convert from that to HFmode (phew!).
6194;; Note that the saturation to SImode requires the SIMD extensions. If
6195;; we ever need to provide this pattern where the SIMD extensions are not
6196;; available, we would need a different approach.
6197
6198(define_expand "<optab>dihf2"
6199 [(set (match_operand:HF 0 "register_operand")
6200 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6201 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6202{
6203 if (TARGET_FP_F16INST)
6204 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6205 else
6206 {
6207 rtx sat_target = gen_reg_rtx (SImode);
6208 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6209 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6210 }
6211
6212 DONE;
6213}
6214)
6215
3f598afe
JW		 6216;; Convert between fixed-point and floating-point (scalar modes)
6217
6218(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6219 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6220 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6221 (match_operand:SI 2 "immediate_operand" "i, i")]
6222 FCVT_F2FIXED))]
6223 ""
6224 "@
1f0e9e34
JG		 6225 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6226 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
3f598afe 6227 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
488461d8 6228 (set_attr "arch" "fp,simd")]
3f598afe
JW		 6229)
6230
6231(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6232 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6233 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6234 (match_operand:SI 2 "immediate_operand" "i, i")]
6235 FCVT_FIXED2F))]
6236 ""
6237 "@
1f0e9e34
JG		 6238 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6239 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
3f598afe 6240 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
488461d8 6241 (set_attr "arch" "fp,simd")]
3f598afe
JW		 6242)
6243
68ad28c3
JW		 6244(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6245 [(set (match_operand:GPI 0 "register_operand" "=r")
6246 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6247 (match_operand:SI 2 "immediate_operand" "i")]
6248 FCVT_F2FIXED))]
6249 "TARGET_FP_F16INST"
6250 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6251 [(set_attr "type" "f_cvtf2i")]
6252)
6253
6254(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6255 [(set (match_operand:HF 0 "register_operand" "=w")
6256 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6257 (match_operand:SI 2 "immediate_operand" "i")]
6258 FCVT_FIXED2F))]
6259 "TARGET_FP_F16INST"
6260 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6261 [(set_attr "type" "f_cvti2f")]
6262)
6263
6264(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6265 [(set (match_operand:HI 0 "register_operand" "=w")
6266 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6267 (match_operand:SI 2 "immediate_operand" "i")]
6268 FCVT_F2FIXED))]
6269 "TARGET_SIMD"
6270 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6271 [(set_attr "type" "neon_fp_to_int_s")]
6272)
6273
6274(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6275 [(set (match_operand:HF 0 "register_operand" "=w")
6276 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6277 (match_operand:SI 2 "immediate_operand" "i")]
6278 FCVT_FIXED2F))]
6279 "TARGET_SIMD"
6280 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6281 [(set_attr "type" "neon_int_to_fp_s")]
6282)
6283
43e9d192
IB		 6284;; -------------------------------------------------------------------
6285;; Floating-point arithmetic
6286;; -------------------------------------------------------------------
6287
6288(define_insn "add<mode>3"
68ad28c3
JW		 6289 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6290 (plus:GPF_F16
6291 (match_operand:GPF_F16 1 "register_operand" "w")
6292 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6293 "TARGET_FLOAT"
6294 "fadd\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6295 [(set_attr "type" "fadd<stype>")]
43e9d192
IB		 6296)
6297
6298(define_insn "sub<mode>3"
68ad28c3
JW		 6299 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6300 (minus:GPF_F16
6301 (match_operand:GPF_F16 1 "register_operand" "w")
6302 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6303 "TARGET_FLOAT"
6304 "fsub\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6305 [(set_attr "type" "fadd<stype>")]
43e9d192
IB		 6306)
6307
6308(define_insn "mul<mode>3"
68ad28c3
JW		 6309 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6310 (mult:GPF_F16
6311 (match_operand:GPF_F16 1 "register_operand" "w")
6312 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6313 "TARGET_FLOAT"
6314 "fmul\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6315 [(set_attr "type" "fmul<stype>")]
43e9d192
IB		 6316)
6317
6318(define_insn "*fnmul<mode>3"
6319 [(set (match_operand:GPF 0 "register_operand" "=w")
6320 (mult:GPF
6321 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6322 (match_operand:GPF 2 "register_operand" "w")))]
2b3fe2e9
SN		 6323 "TARGET_FLOAT && !flag_rounding_math"
6324 "fnmul\\t%<s>0, %<s>1, %<s>2"
6325 [(set_attr "type" "fmul<s>")]
6326)
6327
6328(define_insn "*fnmul<mode>3"
6329 [(set (match_operand:GPF 0 "register_operand" "=w")
6330 (neg:GPF (mult:GPF
6331 (match_operand:GPF 1 "register_operand" "w")
6332 (match_operand:GPF 2 "register_operand" "w"))))]
43e9d192
IB		 6333 "TARGET_FLOAT"
6334 "fnmul\\t%<s>0, %<s>1, %<s>2"
99e6af15 6335 [(set_attr "type" "fmul<s>")]
43e9d192
IB		 6336)
6337
79a2bc2d 6338(define_expand "div<mode>3"
68ad28c3
JW		 6339 [(set (match_operand:GPF_F16 0 "register_operand")
6340 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6341 (match_operand:GPF_F16 2 "register_operand")))]
1be49a38 6342 "TARGET_FLOAT"
79a2bc2d
EM		 6343{
6344 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6345 DONE;
6346
6347 operands[1] = force_reg (<MODE>mode, operands[1]);
6348})
6349
6350(define_insn "*div<mode>3"
68ad28c3
JW		 6351 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6352 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6353 (match_operand:GPF_F16 2 "register_operand" "w")))]
43e9d192
IB		 6354 "TARGET_FLOAT"
6355 "fdiv\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6356 [(set_attr "type" "fdiv<stype>")]
43e9d192
IB		 6357)
6358
6359(define_insn "neg<mode>2"
d7f33f07
JW		 6360 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6361 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6362 "TARGET_FLOAT"
6363 "fneg\\t%<s>0, %<s>1"
d7f33f07 6364 [(set_attr "type" "ffarith<stype>")]
43e9d192
IB		 6365)
6366
98daafa0 6367(define_expand "sqrt<mode>2"
1bbffb87
DZ		 6368 [(set (match_operand:GPF_F16 0 "register_operand")
6369 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
98daafa0
EM		 6370 "TARGET_FLOAT"
6371{
6372 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6373 DONE;
6374})
6375
6376(define_insn "*sqrt<mode>2"
d7f33f07
JW		 6377 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6378 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6379 "TARGET_FLOAT"
6380 "fsqrt\\t%<s>0, %<s>1"
d7f33f07 6381 [(set_attr "type" "fsqrt<stype>")]
43e9d192
IB		 6382)
6383
6384(define_insn "abs<mode>2"
d7f33f07
JW		 6385 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6386 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
43e9d192
IB		 6387 "TARGET_FLOAT"
6388 "fabs\\t%<s>0, %<s>1"
d7f33f07 6389 [(set_attr "type" "ffarith<stype>")]
43e9d192
IB		 6390)
6391
6392;; Given that smax/smin do not specify the result when either input is NaN,
6393;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6394;; for smin.
6395
6396(define_insn "smax<mode>3"
6397 [(set (match_operand:GPF 0 "register_operand" "=w")
6398 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6399 (match_operand:GPF 2 "register_operand" "w")))]
6400 "TARGET_FLOAT"
6401 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
99e6af15 6402 [(set_attr "type" "f_minmax<s>")]
43e9d192
IB		 6403)
6404
6405(define_insn "smin<mode>3"
6406 [(set (match_operand:GPF 0 "register_operand" "=w")
6407 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6408 (match_operand:GPF 2 "register_operand" "w")))]
6409 "TARGET_FLOAT"
6410 "fminnm\\t%<s>0, %<s>1, %<s>2"
99e6af15 6411 [(set_attr "type" "f_minmax<s>")]
43e9d192
IB		 6412)
6413
1efafef3
TC		 6414;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6415;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6416;; which implement the IEEE fmax ()/fmin () functions.
6417(define_insn "<maxmin_uns><mode>3"
68ad28c3
JW		 6418 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6419 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6420 (match_operand:GPF_F16 2 "register_operand" "w")]
1efafef3 6421 FMAXMIN_UNS))]
202d0c11 6422 "TARGET_FLOAT"
1efafef3 6423 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
68ad28c3 6424 [(set_attr "type" "f_minmax<stype>")]
202d0c11
DS		 6425)
6426
f66e6423
TC		 6427(define_expand "lrint<GPF:mode><GPI:mode>2"
6428 [(match_operand:GPI 0 "register_operand")
6429 (match_operand:GPF 1 "register_operand")]
1013465f 6430 "TARGET_FLOAT
fb802d91 6431 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
1013465f 6432 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
f66e6423
TC		 6433{
6434 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6435 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6436 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6437 DONE;
6438}
6439)
6440
fb0f04fe
JG		 6441;; For copysign (x, y), we want to generate:
6442;;
6443;; LDR d2, #(1 << 63)
6444;; BSL v2.8b, [y], [x]
6445;;
315fdae8
RE		 6446;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because
6447;; we expect these operations to nearly always operate on
6448;; floating-point values, we do not want the operation to be
6449;; simplified into a bit-field insert operation that operates on the
6450;; integer side, since typically that would involve three inter-bank
6451;; register copies. As we do not expect copysign to be followed by
6452;; other logical operations on the result, it seems preferable to keep
6453;; this as an unspec operation, rather than exposing the underlying
6454;; logic to the compiler.
6455
6456(define_expand "copysign<GPF:mode>3"
6457 [(match_operand:GPF 0 "register_operand")
6458 (match_operand:GPF 1 "register_operand")
6459 (match_operand:GPF 2 "register_operand")]
fb0f04fe
JG		 6460 "TARGET_FLOAT && TARGET_SIMD"
6461{
315fdae8
RE		 6462 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6463 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6464 << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6465 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6466 bitmask));
fb0f04fe
JG		 6467 DONE;
6468}
6469)
6470
315fdae8
RE		 6471(define_insn "copysign<GPF:mode>3_insn"
6472 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6473 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6474 (match_operand:GPF 2 "register_operand" "w,w,0,0")
6475 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6476 UNSPEC_COPYSIGN))]
fb0f04fe 6477 "TARGET_FLOAT && TARGET_SIMD"
315fdae8
RE		 6478 "@
6479 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6480 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6481 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6482 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6483 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
fb0f04fe
JG		 6484)
6485
315fdae8 6486
4261463d
TC		 6487;; For xorsign (x, y), we want to generate:
6488;;
6489;; LDR d2, #1<<63
6490;; AND v3.8B, v1.8B, v2.8B
6491;; EOR v0.8B, v0.8B, v3.8B
6492;;
6493
6494(define_expand "xorsign<mode>3"
6495 [(match_operand:GPF 0 "register_operand")
6496 (match_operand:GPF 1 "register_operand")
6497 (match_operand:GPF 2 "register_operand")]
6498 "TARGET_FLOAT && TARGET_SIMD"
6499{
6500
5f565314 6501 machine_mode imode = <V_INT_EQUIV>mode;
4261463d
TC		 6502 rtx mask = gen_reg_rtx (imode);
6503 rtx op1x = gen_reg_rtx (imode);
6504 rtx op2x = gen_reg_rtx (imode);
6505
6506 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6507 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6508 imode)));
6509
5f565314
RS		 6510 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6511 lowpart_subreg (imode, operands[2],
6512 <MODE>mode)));
6513 emit_insn (gen_xor<v_int_equiv>3 (op1x,
6514 lowpart_subreg (imode, operands[1],
6515 <MODE>mode),
6516 op2x));
4261463d
TC		 6517 emit_move_insn (operands[0],
6518 lowpart_subreg (<MODE>mode, op1x, imode));
6519 DONE;
6520}
6521)
6522
43e9d192
IB		 6523;; -------------------------------------------------------------------
6524;; Reload support
6525;; -------------------------------------------------------------------
b4f50fd4
RR		 6526;; Reload Scalar Floating point modes from constant pool.
6527;; The AArch64 port doesn't have __int128 constant move support.
1bbffb87 6528;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
0016d8d9 6529(define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
b4f50fd4
RR		 6530 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6531 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6532 (clobber (match_operand:P 2 "register_operand" "=&r"))]
197d1c09 6533 "TARGET_FLOAT"
b4f50fd4
RR		 6534 {
6535 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6536 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6537 DONE;
6538 }
6539)
6540
6541;; Reload Vector modes from constant pool.
0016d8d9 6542(define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
b4f50fd4
RR		 6543 [(set (match_operand:VALL 0 "register_operand" "=w")
6544 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6545 (clobber (match_operand:P 2 "register_operand" "=&r"))]
197d1c09 6546 "TARGET_FLOAT"
b4f50fd4
RR		 6547 {
6548 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6549 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6550 DONE;
6551 }
6552)
43e9d192 6553
0016d8d9 6554(define_expand "@aarch64_reload_mov<mode>"
43e9d192
IB		 6555 [(set (match_operand:TX 0 "register_operand" "=w")
6556 (match_operand:TX 1 "register_operand" "w"))
6557 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6558 ]
a636bb40 6559 "TARGET_FLOAT"
43e9d192
IB		 6560 {
6561 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6562 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6563 gen_aarch64_movtilow_tilow (op0, op1);
6564 gen_aarch64_movdi_tihigh (operands[2], op1);
6565 gen_aarch64_movtihigh_di (op0, operands[2]);
6566 DONE;
6567 }
6568)
6569
6570;; The following secondary reload helpers patterns are invoked
6571;; after or during reload as we don't want these patterns to start
6572;; kicking in during the combiner.
2a9704d0 6573
0016d8d9 6574(define_insn "@aarch64_movdi_<mode>low"
43e9d192 6575 [(set (match_operand:DI 0 "register_operand" "=r")
4c81f958
JG		 6576 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6577 (const_int 64) (const_int 0)))]
a636bb40 6578 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6579 "fmov\\t%x0, %d1"
c1dfcec7 6580 [(set_attr "type" "f_mrc")
43e9d192
IB		 6581 (set_attr "length" "4")
6582 ])
6583
0016d8d9 6584(define_insn "@aarch64_movdi_<mode>high"
43e9d192 6585 [(set (match_operand:DI 0 "register_operand" "=r")
4c81f958
JG		 6586 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6587 (const_int 64) (const_int 64)))]
a636bb40 6588 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6589 "fmov\\t%x0, %1.d[1]"
c1dfcec7 6590 [(set_attr "type" "f_mrc")
43e9d192
IB		 6591 (set_attr "length" "4")
6592 ])
6593
0016d8d9 6594(define_insn "@aarch64_mov<mode>high_di"
12dc6974 6595 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
43e9d192 6596 (const_int 64) (const_int 64))
12dc6974 6597 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
a636bb40 6598 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6599 "fmov\\t%0.d[1], %x1"
c1dfcec7 6600 [(set_attr "type" "f_mcr")
43e9d192
IB		 6601 (set_attr "length" "4")
6602 ])
6603
0016d8d9 6604(define_insn "@aarch64_mov<mode>low_di"
12dc6974
SN		 6605 [(set (match_operand:TX 0 "register_operand" "=w")
6606 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
a636bb40 6607 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6608 "fmov\\t%d0, %x1"
c1dfcec7 6609 [(set_attr "type" "f_mcr")
43e9d192
IB		 6610 (set_attr "length" "4")
6611 ])
6612
6613(define_insn "aarch64_movtilow_tilow"
6614 [(set (match_operand:TI 0 "register_operand" "=w")
a636bb40 6615 (zero_extend:TI
43e9d192 6616 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
a636bb40 6617 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
43e9d192 6618 "fmov\\t%d0, %d1"
d7256bb8 6619 [(set_attr "type" "fmov")
43e9d192
IB		 6620 (set_attr "length" "4")
6621 ])
6622
6623;; There is a deliberate reason why the parameters of high and lo_sum's
6624;; don't have modes for ADRP and ADD instructions. This is to allow high
6625;; and lo_sum's to be used with the labels defining the jump tables in
6626;; rodata section.
6627
28514dda 6628(define_expand "add_losym"
1bbffb87
DZ		 6629 [(set (match_operand 0 "register_operand")
6630 (lo_sum (match_operand 1 "register_operand")
6631 (match_operand 2 "aarch64_valid_symref")))]
43e9d192 6632 ""
28514dda 6633{
ef4bddc2 6634 machine_mode mode = GET_MODE (operands[0]);
28514dda
YZ		 6635
6636 emit_insn ((mode == DImode
6637 ? gen_add_losym_di
6638 : gen_add_losym_si) (operands[0],
6639 operands[1],
6640 operands[2]));
6641 DONE;
6642})
6643
6644(define_insn "add_losym_<mode>"
6645 [(set (match_operand:P 0 "register_operand" "=r")
6646 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6647 (match_operand 2 "aarch64_valid_symref" "S")))]
6648 ""
d6591257 6649 "add\\t%<w>0, %<w>1, :lo12:%c2"
d8e11262 6650 [(set_attr "type" "alu_imm")]
28514dda 6651)
43e9d192 6652
28514dda
YZ		 6653(define_insn "ldr_got_small_<mode>"
6654 [(set (match_operand:PTR 0 "register_operand" "=r")
6655 (unspec:PTR [(mem:PTR (lo_sum:PTR
6656 (match_operand:PTR 1 "register_operand" "r")
6657 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6658 UNSPEC_GOTSMALLPIC))]
6659 ""
d6591257 6660 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
db46a2e6 6661 [(set_attr "type" "load_<ldst_sz>")]
43e9d192
IB		 6662)
6663
28514dda 6664(define_insn "ldr_got_small_sidi"
43e9d192 6665 [(set (match_operand:DI 0 "register_operand" "=r")
28514dda
YZ		 6666 (zero_extend:DI
6667 (unspec:SI [(mem:SI (lo_sum:DI
6668 (match_operand:DI 1 "register_operand" "r")
6669 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6670 UNSPEC_GOTSMALLPIC)))]
6671 "TARGET_ILP32"
d6591257 6672 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
89b2133e 6673 [(set_attr "type" "load_4")]
43e9d192
IB		 6674)
6675
1b1e81f8
JW		 6676(define_insn "ldr_got_small_28k_<mode>"
6677 [(set (match_operand:PTR 0 "register_operand" "=r")
6678 (unspec:PTR [(mem:PTR (lo_sum:PTR
6679 (match_operand:PTR 1 "register_operand" "r")
6680 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6681 UNSPEC_GOTSMALLPIC28K))]
6682 ""
d6591257 6683 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
db46a2e6 6684 [(set_attr "type" "load_<ldst_sz>")]
1b1e81f8
JW		 6685)
6686
6687(define_insn "ldr_got_small_28k_sidi"
6688 [(set (match_operand:DI 0 "register_operand" "=r")
6689 (zero_extend:DI
6690 (unspec:SI [(mem:SI (lo_sum:DI
6691 (match_operand:DI 1 "register_operand" "r")
6692 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6693 UNSPEC_GOTSMALLPIC28K)))]
6694 "TARGET_ILP32"
d6591257 6695 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
89b2133e 6696 [(set_attr "type" "load_4")]
1b1e81f8
JW		 6697)
6698
87dd8ab0
MS		 6699(define_insn "ldr_got_tiny"
6700 [(set (match_operand:DI 0 "register_operand" "=r")
6701 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6702 UNSPEC_GOTTINYPIC))]
6703 ""
6704 "ldr\\t%0, %L1"
db46a2e6 6705 [(set_attr "type" "load_8")]
87dd8ab0
MS		 6706)
6707
43e9d192
IB		 6708(define_insn "aarch64_load_tp_hard"
6709 [(set (match_operand:DI 0 "register_operand" "=r")
6710 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6711 ""
6712 "mrs\\t%0, tpidr_el0"
99e6af15 6713 [(set_attr "type" "mrs")]
43e9d192
IB		 6714)
6715
6716;; The TLS ABI specifically requires that the compiler does not schedule
6717;; instructions in the TLS stubs, in order to enable linker relaxation.
6718;; Therefore we treat the stubs as an atomic sequence.
23b88fda 6719(define_expand "tlsgd_small_<mode>"
1bbffb87 6720 [(parallel [(set (match_operand 0 "register_operand")
43e9d192 6721 (call (mem:DI (match_dup 2)) (const_int 1)))
08cc4d92 6722 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
1bbffb87 6723 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
43e9d192
IB		 6724 (clobber (reg:DI LR_REGNUM))])]
6725 ""
6726{
6727 operands[2] = aarch64_tls_get_addr ();
6728})
6729
23b88fda 6730(define_insn "*tlsgd_small_<mode>"
43e9d192
IB		 6731 [(set (match_operand 0 "register_operand" "")
6732 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
08cc4d92 6733 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
23b88fda 6734 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
43e9d192
IB		 6735 (clobber (reg:DI LR_REGNUM))
6736 ]
6737 ""
6738 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
c1dfcec7 6739 [(set_attr "type" "call")
43e9d192
IB		 6740 (set_attr "length" "16")])
6741
621ad2de
AP		 6742(define_insn "tlsie_small_<mode>"
6743 [(set (match_operand:PTR 0 "register_operand" "=r")
6744 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
43e9d192
IB		 6745 UNSPEC_GOTSMALLTLS))]
6746 ""
621ad2de 6747 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
89b2133e 6748 [(set_attr "type" "load_4")
43e9d192
IB		 6749 (set_attr "length" "8")]
6750)
6751
621ad2de 6752(define_insn "tlsie_small_sidi"
43e9d192 6753 [(set (match_operand:DI 0 "register_operand" "=r")
621ad2de
AP		 6754 (zero_extend:DI
6755 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6756 UNSPEC_GOTSMALLTLS)))]
6757 ""
6758 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
89b2133e 6759 [(set_attr "type" "load_4")
621ad2de
AP		 6760 (set_attr "length" "8")]
6761)
6762
5ae7caad
JW		 6763(define_insn "tlsie_tiny_<mode>"
6764 [(set (match_operand:PTR 0 "register_operand" "=&r")
6765 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6766 (match_operand:PTR 2 "register_operand" "r")]
6767 UNSPEC_GOTTINYTLS))]
6768 ""
6769 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6770 [(set_attr "type" "multiple")
6771 (set_attr "length" "8")]
6772)
6773
6774(define_insn "tlsie_tiny_sidi"
6775 [(set (match_operand:DI 0 "register_operand" "=&r")
6776 (zero_extend:DI
6777 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6778 (match_operand:DI 2 "register_operand" "r")
6779 ]
6780 UNSPEC_GOTTINYTLS)))]
6781 ""
5b295a81 6782 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5ae7caad
JW		 6783 [(set_attr "type" "multiple")
6784 (set_attr "length" "8")]
6785)
6786
cbf5629e
JW		 6787(define_insn "tlsle12_<mode>"
6788 [(set (match_operand:P 0 "register_operand" "=r")
6789 (unspec:P [(match_operand:P 1 "register_operand" "r")
6790 (match_operand 2 "aarch64_tls_le_symref" "S")]
6791 UNSPEC_TLSLE12))]
621ad2de 6792 ""
cbf5629e
JW		 6793 "add\\t%<w>0, %<w>1, #%L2";
6794 [(set_attr "type" "alu_sreg")
6795 (set_attr "length" "4")]
6796)
621ad2de 6797
cbf5629e 6798(define_insn "tlsle24_<mode>"
621ad2de 6799 [(set (match_operand:P 0 "register_operand" "=r")
cbf5629e
JW		 6800 (unspec:P [(match_operand:P 1 "register_operand" "r")
6801 (match_operand 2 "aarch64_tls_le_symref" "S")]
6802 UNSPEC_TLSLE24))]
43e9d192 6803 ""
0699caae 6804 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
cbf5629e 6805 [(set_attr "type" "multiple")
43e9d192
IB		 6806 (set_attr "length" "8")]
6807)
6808
cbf5629e
JW		 6809(define_insn "tlsle32_<mode>"
6810 [(set (match_operand:P 0 "register_operand" "=r")
6811 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6812 UNSPEC_TLSLE32))]
6813 ""
6814 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6815 [(set_attr "type" "multiple")
6816 (set_attr "length" "8")]
6817)
6818
6819(define_insn "tlsle48_<mode>"
6820 [(set (match_operand:P 0 "register_operand" "=r")
6821 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6822 UNSPEC_TLSLE48))]
6823 ""
6824 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6825 [(set_attr "type" "multiple")
6826 (set_attr "length" "12")]
6827)
6828
80c13ac5
RS		 6829(define_expand "tlsdesc_small_<mode>"
6830 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6831 "TARGET_TLS_DESC"
6832 {
6833 if (TARGET_SVE)
bb6ce448
RS		 6834 {
6835 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode);
6836 rtx_insn *call
6837 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
6838 RTL_CONST_CALL_P (call) = 1;
6839 }
80c13ac5
RS		 6840 else
6841 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6842 DONE;
6843 }
6844)
6845
6846;; tlsdesc calls preserve all core and Advanced SIMD registers except
6847;; R0 and LR.
6848(define_insn "tlsdesc_small_advsimd_<mode>"
621ad2de
AP		 6849 [(set (reg:PTR R0_REGNUM)
6850 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
80c13ac5 6851 UNSPEC_TLSDESC))
43e9d192 6852 (clobber (reg:DI LR_REGNUM))
0bc11714 6853 (clobber (reg:CC CC_REGNUM))
0e510d18
ST		 6854 (clobber (match_scratch:DI 1 "=r"))
6855 (use (reg:DI FP_REGNUM))]
80c13ac5
RS		 6856 "TARGET_TLS_DESC && !TARGET_SVE"
6857 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6858 [(set_attr "type" "call")
6859 (set_attr "length" "16")])
6860
bb6ce448
RS		 6861;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
6862;; describing the extra call-preserved guarantees. This would work
6863;; for non-SVE too, but avoiding a call is probably better if we can.
80c13ac5
RS		 6864(define_insn "tlsdesc_small_sve_<mode>"
6865 [(set (reg:PTR R0_REGNUM)
bb6ce448
RS		 6866 (call (mem:DI (unspec:PTR
6867 [(match_operand 0 "aarch64_valid_symref")]
6868 UNSPEC_TLSDESC))
6869 (const_int 0)))
6870 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
80c13ac5 6871 (clobber (reg:DI LR_REGNUM))
bb6ce448 6872 (clobber (match_scratch:DI 2 "=r"))]
80c13ac5 6873 "TARGET_TLS_DESC && TARGET_SVE"
bb6ce448 6874 "adrp\\tx0, %A0\;ldr\\t%<w>2, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%2"
c1dfcec7 6875 [(set_attr "type" "call")
43e9d192
IB		 6876 (set_attr "length" "16")])
6877
6878(define_insn "stack_tie"
6879 [(set (mem:BLK (scratch))
6880 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6881 (match_operand:DI 1 "register_operand" "rk")]
6882 UNSPEC_PRLG_STK))]
6883 ""
6884 ""
6885 [(set_attr "length" "0")]
6886)
6887
e1d5d19e
KT		 6888(define_insn "aarch64_fjcvtzs"
6889 [(set (match_operand:SI 0 "register_operand" "=r")
6890 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
6891 UNSPEC_FJCVTZS))]
6892 "TARGET_JSCVT"
6893 "fjcvtzs\\t%w0, %d1"
6894 [(set_attr "type" "f_cvtf2i")]
6895)
6896
db58fd89
JW		 6897;; Pointer authentication patterns are always provided. In architecture
6898;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6899;; This lets the user write portable software which authenticates pointers
6900;; when run on something which implements ARMv8.3-A, and which runs
6901;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6902;; implemented.
6903
6904;; Signing/Authenticating R30 using SP as the salt.
6905
6906(define_insn "<pauth_mnem_prefix>sp"
6907 [(set (reg:DI R30_REGNUM)
6908 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6909 ""
8fc16d72 6910 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
db58fd89
JW		 6911)
6912
6913;; Signing/Authenticating X17 using X16 as the salt.
6914
6915(define_insn "<pauth_mnem_prefix>1716"
6916 [(set (reg:DI R17_REGNUM)
6917 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6918 ""
8fc16d72 6919 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
db58fd89
JW		 6920)
6921
6922;; Stripping the signature in R30.
6923
6924(define_insn "xpaclri"
6925 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6926 ""
6927 "hint\t7 // xpaclri"
6928)
6929
a3eb8a52
EB		 6930;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6931;; all of memory. This blocks insns from being moved across this point.
6932
6933(define_insn "blockage"
6934 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6935 ""
6936 ""
6937 [(set_attr "length" "0")
6938 (set_attr "type" "block")]
6939)
6940
5f5c5e0f 6941(define_insn "probe_stack_range"
cd1bef27 6942 [(set (match_operand:DI 0 "register_operand" "=rk")
5f5c5e0f
EB		 6943 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6944 (match_operand:DI 2 "register_operand" "r")]
6945 UNSPECV_PROBE_STACK_RANGE))]
a3eb8a52
EB		 6946 ""
6947{
6948 return aarch64_output_probe_stack_range (operands[0], operands[2]);
6949}
6950 [(set_attr "length" "32")]
6951)
6952
eb471ba3
TC		 6953;; This instruction is used to generate the stack clash stack adjustment and
6954;; probing loop. We can't change the control flow during prologue and epilogue
6955;; code generation. So we must emit a volatile unspec and expand it later on.
6956
6957(define_insn "@probe_sve_stack_clash_<mode>"
6958 [(set (match_operand:P 0 "register_operand" "=rk")
6959 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6960 (match_operand:P 2 "register_operand" "r")
6961 (match_operand:P 3 "const_int_operand" "n")
6962 (match_operand:P 4 "aarch64_plus_immediate" "L")]
6963 UNSPECV_PROBE_STACK_RANGE))]
6964 "TARGET_SVE"
6965{
6966 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6967 operands[3], operands[4]);
6968}
6969 [(set_attr "length" "28")]
6970)
6971
a5bc806c
AP		 6972;; Named pattern for expanding thread pointer reference.
6973(define_expand "get_thread_pointerdi"
1bbffb87 6974 [(match_operand:DI 0 "register_operand")]
a5bc806c
AP		 6975 ""
6976{
6977 rtx tmp = aarch64_load_tp (operands[0]);
6978 if (tmp != operands[0])
6979 emit_move_insn (operands[0], tmp);
6980 DONE;
6981})
6982
36e17020
VK		 6983;; Named patterns for stack smashing protection.
6984(define_expand "stack_protect_set"
6985 [(match_operand 0 "memory_operand")
6986 (match_operand 1 "memory_operand")]
6987 ""
6988{
ef4bddc2 6989 machine_mode mode = GET_MODE (operands[0]);
cd0b2d36
RR		 6990 if (aarch64_stack_protector_guard != SSP_GLOBAL)
6991 {
6992 /* Generate access through the system register. */
6993 rtx tmp_reg = gen_reg_rtx (mode);
6994 if (mode == DImode)
6995 {
6996 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6997 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6998 GEN_INT (aarch64_stack_protector_guard_offset)));
6999 }
7000 else
7001 {
7002 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7003 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7004 GEN_INT (aarch64_stack_protector_guard_offset)));
36e17020 7005
cd0b2d36
RR		 7006 }
7007 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7008 }
7009
36e17020
VK		 7010 emit_insn ((mode == DImode
7011 ? gen_stack_protect_set_di
7012 : gen_stack_protect_set_si) (operands[0], operands[1]));
7013 DONE;
7014})
7015
cd0b2d36
RR		 7016(define_insn "reg_stack_protect_address_<mode>"
7017 [(set (match_operand:PTR 0 "register_operand" "=r")
7018 (unspec:PTR [(const_int 0)]
7019 UNSPEC_SSP_SYSREG))]
7020 "aarch64_stack_protector_guard != SSP_GLOBAL"
7021 {
7022 char buf[150];
7023 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7024 aarch64_stack_protector_guard_reg_str);
7025 output_asm_insn (buf, operands);
7026 return "";
7027 }
7028 [(set_attr "type" "mrs")])
7029
a7e73b41
RE		 7030;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
7031;; canary value does not live beyond the life of this sequence.
36e17020
VK		 7032(define_insn "stack_protect_set_<mode>"
7033 [(set (match_operand:PTR 0 "memory_operand" "=m")
7034 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7035 UNSPEC_SP_SET))
7036 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7037 ""
a7e73b41 7038 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
36e17020
VK		 7039 [(set_attr "length" "12")
7040 (set_attr "type" "multiple")])
7041
7042(define_expand "stack_protect_test"
7043 [(match_operand 0 "memory_operand")
7044 (match_operand 1 "memory_operand")
7045 (match_operand 2)]
7046 ""
7047{
9c6ab05f 7048 rtx result;
ef4bddc2 7049 machine_mode mode = GET_MODE (operands[0]);
36e17020 7050
9c6ab05f 7051 result = gen_reg_rtx(mode);
cd0b2d36
RR		 7052 if (aarch64_stack_protector_guard != SSP_GLOBAL)
7053 {
7054 /* Generate access through the system register. The
7055 sequence we want here is the access
7056 of the stack offset to come with
7057 mrs scratch_reg, <system_register>
7058 add scratch_reg, scratch_reg, :lo12:offset. */
7059 rtx tmp_reg = gen_reg_rtx (mode);
7060 if (mode == DImode)
7061 {
7062 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7063 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7064 GEN_INT (aarch64_stack_protector_guard_offset)));
7065 }
7066 else
7067 {
7068 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7069 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7070 GEN_INT (aarch64_stack_protector_guard_offset)));
9c6ab05f 7071
cd0b2d36
RR		 7072 }
7073 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7074 }
36e17020 7075 emit_insn ((mode == DImode
cd0b2d36
RR		 7076 ? gen_stack_protect_test_di
7077 : gen_stack_protect_test_si) (result,
7078 operands[0],
7079 operands[1]));
36e17020
VK		 7080
7081 if (mode == DImode)
7082 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7083 result, const0_rtx, operands[2]));
7084 else
7085 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7086 result, const0_rtx, operands[2]));
7087 DONE;
7088})
7089
7090(define_insn "stack_protect_test_<mode>"
2ce145f5 7091 [(set (match_operand:PTR 0 "register_operand" "=r")
36e17020
VK		 7092 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7093 (match_operand:PTR 2 "memory_operand" "m")]
7094 UNSPEC_SP_TEST))
b4e9251d 7095 (clobber (match_scratch:PTR 3 "=&r"))]
36e17020 7096 ""
954224d7 7097 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
36e17020
VK		 7098 [(set_attr "length" "12")
7099 (set_attr "type" "multiple")])
7100
aa87aced
KV		 7101;; Write Floating-point Control Register.
7102(define_insn "set_fpcr"
7103 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7104 ""
a2adad8b 7105 "msr\\tfpcr, %0"
aa87aced
KV		 7106 [(set_attr "type" "mrs")])
7107
7108;; Read Floating-point Control Register.
7109(define_insn "get_fpcr"
7110 [(set (match_operand:SI 0 "register_operand" "=r")
7111 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7112 ""
7113 "mrs\\t%0, fpcr"
7114 [(set_attr "type" "mrs")])
7115
7116;; Write Floating-point Status Register.
7117(define_insn "set_fpsr"
7118 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7119 ""
7120 "msr\\tfpsr, %0"
7121 [(set_attr "type" "mrs")])
7122
7123;; Read Floating-point Status Register.
7124(define_insn "get_fpsr"
7125 [(set (match_operand:SI 0 "register_operand" "=r")
7126 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7127 ""
7128 "mrs\\t%0, fpsr"
7129 [(set_attr "type" "mrs")])
7130
7131
594bdd53
FY		 7132;; Define the subtract-one-and-jump insns so loop.c
7133;; knows what to generate.
7134(define_expand "doloop_end"
7135 [(use (match_operand 0 "" "")) ; loop pseudo
7136 (use (match_operand 1 "" ""))] ; label
7137 "optimize > 0 && flag_modulo_sched"
7138{
7139 rtx s0;
7140 rtx bcomp;
7141 rtx loc_ref;
7142 rtx cc_reg;
7143 rtx insn;
7144 rtx cmp;
7145
7146 /* Currently SMS relies on the do-loop pattern to recognize loops
7147 where (1) the control part consists of all insns defining and/or
7148 using a certain 'count' register and (2) the loop count can be
7149 adjusted by modifying this register prior to the loop.
7150 ??? The possible introduction of a new block to initialize the
7151 new IV can potentially affect branch optimizations. */
7152
7153 if (GET_MODE (operands[0]) != DImode)
7154 FAIL;
7155
7156 s0 = operands [0];
7157 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7158
7159 cmp = XVECEXP (PATTERN (insn), 0, 0);
7160 cc_reg = SET_DEST (cmp);
7161 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7162 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
f7df4a84 7163 emit_jump_insn (gen_rtx_SET (pc_rtx,
594bdd53
FY		 7164 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7165 loc_ref, pc_rtx)));
7166 DONE;
7167})
7168
3751345d
RE		 7169;; Track speculation through conditional branches. We assume that
7170;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7171(define_insn "speculation_tracker"
7172 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
996bea89 7173 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
3751345d
RE		 7174 UNSPEC_SPECULATION_TRACKER))]
7175 ""
7176 {
7177 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7178 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7179 return "";
7180 }
7181 [(set_attr "type" "csel")]
7182)
7183
b5f794b4
SD		 7184;; BTI <target> instructions
7185(define_insn "bti_noarg"
7186 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7187 ""
7188 "hint\t32 // bti"
7189 [(set_attr "type" "no_insn")]
7190)
7191
7192(define_insn "bti_c"
7193 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7194 ""
7195 "hint\t34 // bti c"
7196 [(set_attr "type" "no_insn")]
7197)
7198
7199(define_insn "bti_j"
7200 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7201 ""
7202 "hint\t36 // bti j"
7203 [(set_attr "type" "no_insn")]
7204)
7205
7206(define_insn "bti_jc"
7207 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7208 ""
7209 "hint\t38 // bti jc"
7210 [(set_attr "type" "no_insn")]
7211)
7212
def745b2
RE		 7213;; Hard speculation barrier.
7214(define_insn "speculation_barrier"
7215 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7216 ""
7217 "isb\;dsb\\tsy"
7218 [(set_attr "length" "8")
3751345d
RE		 7219 (set_attr "type" "block")
7220 (set_attr "speculation_barrier" "true")]
def745b2
RE		 7221)
7222
c0111dc4
RE		 7223;; Support for __builtin_speculation_safe_value when we have speculation
7224;; tracking enabled. Use the speculation tracker to decide whether to
7225;; copy operand 1 to the target, or to copy the fail value (operand 2).
21cebf90 7226(define_expand "@despeculate_copy<ALLI_TI:mode>"
1bbffb87 7227 [(set (match_operand:ALLI_TI 0 "register_operand")
c0111dc4 7228 (unspec_volatile:ALLI_TI
1bbffb87
DZ		 7229 [(match_operand:ALLI_TI 1 "register_operand")
7230 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
c0111dc4
RE		 7231 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7232 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7233 ""
7234 "
7235 {
7236 if (operands[2] == const0_rtx)
7237 {
7238 rtx tracker;
7239 if (<MODE>mode == TImode)
7240 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7241 else
7242 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7243
7244 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7245 tracker));
7246 DONE;
7247 }
7248 }
7249 "
7250)
7251
7252;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
7253;; encoding for CSDB, but will work in older versions of the assembler.
7254(define_insn "*despeculate_copy<ALLI:mode>_insn"
7255 [(set (match_operand:ALLI 0 "register_operand" "=r")
7256 (unspec_volatile:ALLI
7257 [(match_operand:ALLI 1 "register_operand" "r")
7258 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7259 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7260 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7261 ""
7262 {
7263 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7264 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7265 operands);
7266 return "";
7267 }
7268 [(set_attr "length" "12")
7269 (set_attr "type" "block")
7270 (set_attr "speculation_barrier" "true")]
7271)
7272
7273;; Pattern to match despeculate_copyti
7274(define_insn "*despeculate_copyti_insn"
7275 [(set (match_operand:TI 0 "register_operand" "=r")
7276 (unspec_volatile:TI
7277 [(match_operand:TI 1 "register_operand" "r")
7278 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7279 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7280 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7281 ""
7282 {
7283 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7284 output_asm_insn
7285 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7286 operands);
7287 return "";
7288 }
7289 [(set_attr "length" "16")
7290 (set_attr "type" "block")
7291 (set_attr "speculation_barrier" "true")]
7292)
7293
7294(define_insn "despeculate_simple<ALLI:mode>"
7295 [(set (match_operand:ALLI 0 "register_operand" "=r")
7296 (unspec_volatile:ALLI
7297 [(match_operand:ALLI 1 "register_operand" "r")
7298 (use (match_operand:ALLI 2 "register_operand" ""))]
7299 UNSPECV_SPECULATION_BARRIER))]
7300 ""
7301 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7302 [(set_attr "type" "block")
7303 (set_attr "length" "8")
7304 (set_attr "speculation_barrier" "true")]
7305)
7306
7307(define_insn "despeculate_simpleti"
7308 [(set (match_operand:TI 0 "register_operand" "=r")
7309 (unspec_volatile:TI
7310 [(match_operand:TI 1 "register_operand" "r")
7311 (use (match_operand:DI 2 "register_operand" ""))]
7312 UNSPECV_SPECULATION_BARRIER))]
7313 ""
7314 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7315 [(set_attr "type" "block")
7316 (set_attr "length" "12")
7317 (set_attr "speculation_barrier" "true")]
7318)
7319
10bd1d96
KT		 7320(define_insn "aarch64_<frintnzs_op><mode>"
7321 [(set (match_operand:VSFDF 0 "register_operand" "=w")
7322 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7323 FRINTNZX))]
7324 "TARGET_FRINT && TARGET_FLOAT
7325 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7326 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7327 [(set_attr "type" "f_rint<stype>")]
7328)
7329
89626179
SD		 7330;; Transactional Memory Extension (TME) instructions.
7331
7332(define_insn "tstart"
7333 [(set (match_operand:DI 0 "register_operand" "=r")
7334 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7335 (clobber (mem:BLK (scratch)))]
7336 "TARGET_TME"
7337 "tstart\\t%0"
7338 [(set_attr "type" "tme")]
7339)
7340
7341(define_insn "ttest"
7342 [(set (match_operand:DI 0 "register_operand" "=r")
7343 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7344 (clobber (mem:BLK (scratch)))]
7345 "TARGET_TME"
7346 "ttest\\t%0"
7347 [(set_attr "type" "tme")]
7348)
7349
7350(define_insn "tcommit"
7351 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7352 (clobber (mem:BLK (scratch)))]
7353 "TARGET_TME"
7354 "tcommit"
7355 [(set_attr "type" "tme")]
7356)
7357
7358(define_insn "tcancel"
7359 [(unspec_volatile:BLK
7360 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7361 (clobber (mem:BLK (scratch)))]
7362 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7363 "tcancel\\t#%0"
7364 [(set_attr "type" "tme")]
7365)
7366
c5dc215d
KT		 7367(define_insn "aarch64_rndr"
7368 [(set (match_operand:DI 0 "register_operand" "=r")
7369 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR))
7370 (set (reg:CC_Z CC_REGNUM)
7371 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))]
7372 "TARGET_RNG"
7373 "mrs\t%0, RNDR"
7374 [(set_attr "type" "mrs")]
7375)
7376
7377(define_insn "aarch64_rndrrs"
7378 [(set (match_operand:DI 0 "register_operand" "=r")
7379 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS))
7380 (set (reg:CC_Z CC_REGNUM)
7381 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))]
7382 "TARGET_RNG"
7383 "mrs\t%0, RNDRRS"
7384 [(set_attr "type" "mrs")]
7385)
7386
43e9d192
IB		 7387;; AdvSIMD Stuff
7388(include "aarch64-simd.md")
7389
0462169c
SN		 7390;; Atomic Operations
7391(include "atomics.md")
350013bc
BC		 7392
7393;; ldp/stp peephole patterns
7394(include "aarch64-ldpstp.md")
43cacb12
RS		 7395
7396;; SVE.
7397(include "aarch64-sve.md")
0617e23c
AM		 7398
7399;; SVE2.
7400(include "aarch64-sve2.md")
GNU Compiler Collection
This page took 3.594348 seconds and 5 git commands to generate.