]> gcc.gnu.org Git - gcc.git/blob - gcc/config/aarch64/aarch64.md
git://gcc.gnu.org / gcc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
[AArch64] Add support for the SVE PCS
[gcc.git] / gcc / config / aarch64 / aarch64.md
1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
20
21 ;; Register numbers
22 (define_constants
23 [
24 (R0_REGNUM 0)
25 (R1_REGNUM 1)
26 (R2_REGNUM 2)
27 (R3_REGNUM 3)
28 (R4_REGNUM 4)
29 (R5_REGNUM 5)
30 (R6_REGNUM 6)
31 (R7_REGNUM 7)
32 (R8_REGNUM 8)
33 (R9_REGNUM 9)
34 (R10_REGNUM 10)
35 (R11_REGNUM 11)
36 (R12_REGNUM 12)
37 (R13_REGNUM 13)
38 (R14_REGNUM 14)
39 (R15_REGNUM 15)
40 (R16_REGNUM 16)
41 (R17_REGNUM 17)
42 (R18_REGNUM 18)
43 (R19_REGNUM 19)
44 (R20_REGNUM 20)
45 (R21_REGNUM 21)
46 (R22_REGNUM 22)
47 (R23_REGNUM 23)
48 (R24_REGNUM 24)
49 (R25_REGNUM 25)
50 (R26_REGNUM 26)
51 (R27_REGNUM 27)
52 (R28_REGNUM 28)
53 (R29_REGNUM 29)
54 (R30_REGNUM 30)
55 (SP_REGNUM 31)
56 (V0_REGNUM 32)
57 (V1_REGNUM 33)
58 (V2_REGNUM 34)
59 (V3_REGNUM 35)
60 (V4_REGNUM 36)
61 (V5_REGNUM 37)
62 (V6_REGNUM 38)
63 (V7_REGNUM 39)
64 (V8_REGNUM 40)
65 (V9_REGNUM 41)
66 (V10_REGNUM 42)
67 (V11_REGNUM 43)
68 (V12_REGNUM 44)
69 (V13_REGNUM 45)
70 (V14_REGNUM 46)
71 (V15_REGNUM 47)
72 (V16_REGNUM 48)
73 (V17_REGNUM 49)
74 (V18_REGNUM 50)
75 (V19_REGNUM 51)
76 (V20_REGNUM 52)
77 (V21_REGNUM 53)
78 (V22_REGNUM 54)
79 (V23_REGNUM 55)
80 (V24_REGNUM 56)
81 (V25_REGNUM 57)
82 (V26_REGNUM 58)
83 (V27_REGNUM 59)
84 (V28_REGNUM 60)
85 (V29_REGNUM 61)
86 (V30_REGNUM 62)
87 (V31_REGNUM 63)
88 (SFP_REGNUM 64)
89 (AP_REGNUM 65)
90 (CC_REGNUM 66)
91 ;; Defined only to make the DWARF description simpler.
92 (VG_REGNUM 67)
93 (P0_REGNUM 68)
94 (P1_REGNUM 69)
95 (P2_REGNUM 70)
96 (P3_REGNUM 71)
97 (P4_REGNUM 72)
98 (P5_REGNUM 73)
99 (P6_REGNUM 74)
100 (P7_REGNUM 75)
101 (P8_REGNUM 76)
102 (P9_REGNUM 77)
103 (P10_REGNUM 78)
104 (P11_REGNUM 79)
105 (P12_REGNUM 80)
106 (P13_REGNUM 81)
107 (P14_REGNUM 82)
108 (P15_REGNUM 83)
109 (LAST_SAVED_REGNUM 83)
110 (FFR_REGNUM 84)
111 ;; "FFR token": a fake register used for representing the scheduling
112 ;; restrictions on FFR-related operations.
113 (FFRT_REGNUM 85)
114 ;; Scratch register used by stack clash protection to calculate
115 ;; SVE CFA offsets during probing.
116 (STACK_CLASH_SVE_CFA_REGNUM 11)
117 ;; Scratch registers for prologue/epilogue use.
118 (EP0_REGNUM 12)
119 (EP1_REGNUM 13)
120 ;; A couple of call-clobbered registers that we need to reserve when
121 ;; tracking speculation this is not ABI, so is subject to change.
122 (SPECULATION_SCRATCH_REGNUM 14)
123 (SPECULATION_TRACKER_REGNUM 15)
124 ;; Scratch registers used in frame layout.
125 (IP0_REGNUM 16)
126 (IP1_REGNUM 17)
127 (FP_REGNUM 29)
128 (LR_REGNUM 30)
129 ]
130 )
131
132 (define_c_enum "unspec" [
133 UNSPEC_AUTIA1716
134 UNSPEC_AUTIB1716
135 UNSPEC_AUTIASP
136 UNSPEC_AUTIBSP
137 UNSPEC_CALLEE_ABI
138 UNSPEC_CASESI
139 UNSPEC_CRC32B
140 UNSPEC_CRC32CB
141 UNSPEC_CRC32CH
142 UNSPEC_CRC32CW
143 UNSPEC_CRC32CX
144 UNSPEC_CRC32H
145 UNSPEC_CRC32W
146 UNSPEC_CRC32X
147 UNSPEC_FCVTZS
148 UNSPEC_FCVTZU
149 UNSPEC_FJCVTZS
150 UNSPEC_FRINT32Z
151 UNSPEC_FRINT32X
152 UNSPEC_FRINT64Z
153 UNSPEC_FRINT64X
154 UNSPEC_URECPE
155 UNSPEC_FRECPE
156 UNSPEC_FRECPS
157 UNSPEC_FRECPX
158 UNSPEC_FRINTA
159 UNSPEC_FRINTI
160 UNSPEC_FRINTM
161 UNSPEC_FRINTN
162 UNSPEC_FRINTP
163 UNSPEC_FRINTX
164 UNSPEC_FRINTZ
165 UNSPEC_GOTSMALLPIC
166 UNSPEC_GOTSMALLPIC28K
167 UNSPEC_GOTSMALLTLS
168 UNSPEC_GOTTINYPIC
169 UNSPEC_GOTTINYTLS
170 UNSPEC_LD1
171 UNSPEC_LD2
172 UNSPEC_LD2_DREG
173 UNSPEC_LD2_DUP
174 UNSPEC_LD3
175 UNSPEC_LD3_DREG
176 UNSPEC_LD3_DUP
177 UNSPEC_LD4
178 UNSPEC_LD4_DREG
179 UNSPEC_LD4_DUP
180 UNSPEC_LD2_LANE
181 UNSPEC_LD3_LANE
182 UNSPEC_LD4_LANE
183 UNSPEC_MB
184 UNSPEC_NOP
185 UNSPEC_PACIA1716
186 UNSPEC_PACIB1716
187 UNSPEC_PACIASP
188 UNSPEC_PACIBSP
189 UNSPEC_PRLG_STK
190 UNSPEC_REV
191 UNSPEC_RBIT
192 UNSPEC_SABAL
193 UNSPEC_SABDL2
194 UNSPEC_SADALP
195 UNSPEC_SCVTF
196 UNSPEC_SISD_NEG
197 UNSPEC_SISD_SSHL
198 UNSPEC_SISD_USHL
199 UNSPEC_SSHL_2S
200 UNSPEC_ST1
201 UNSPEC_ST2
202 UNSPEC_ST3
203 UNSPEC_ST4
204 UNSPEC_ST2_LANE
205 UNSPEC_ST3_LANE
206 UNSPEC_ST4_LANE
207 UNSPEC_TLS
208 UNSPEC_TLSDESC
209 UNSPEC_TLSLE12
210 UNSPEC_TLSLE24
211 UNSPEC_TLSLE32
212 UNSPEC_TLSLE48
213 UNSPEC_UABAL
214 UNSPEC_UABDL2
215 UNSPEC_UADALP
216 UNSPEC_UCVTF
217 UNSPEC_USHL_2S
218 UNSPEC_VSTRUCTDUMMY
219 UNSPEC_SSP_SYSREG
220 UNSPEC_SP_SET
221 UNSPEC_SP_TEST
222 UNSPEC_RSQRT
223 UNSPEC_RSQRTE
224 UNSPEC_RSQRTS
225 UNSPEC_NZCV
226 UNSPEC_XPACLRI
227 UNSPEC_LD1_SVE
228 UNSPEC_ST1_SVE
229 UNSPEC_LDNT1_SVE
230 UNSPEC_STNT1_SVE
231 UNSPEC_LD1RQ
232 UNSPEC_LD1_GATHER
233 UNSPEC_LDFF1_GATHER
234 UNSPEC_ST1_SCATTER
235 UNSPEC_PRED_X
236 UNSPEC_PRED_Z
237 UNSPEC_PTEST
238 UNSPEC_PTRUE
239 UNSPEC_UNPACKSHI
240 UNSPEC_UNPACKUHI
241 UNSPEC_UNPACKSLO
242 UNSPEC_UNPACKULO
243 UNSPEC_PACK
244 UNSPEC_WHILE_LE
245 UNSPEC_WHILE_LO
246 UNSPEC_WHILE_LS
247 UNSPEC_WHILE_LT
248 UNSPEC_LDN
249 UNSPEC_STN
250 UNSPEC_INSR
251 UNSPEC_CLASTA
252 UNSPEC_CLASTB
253 UNSPEC_FADDA
254 UNSPEC_REV_SUBREG
255 UNSPEC_REINTERPRET
256 UNSPEC_SPECULATION_TRACKER
257 UNSPEC_COPYSIGN
258 UNSPEC_TTEST ; Represent transaction test.
259 UNSPEC_UPDATE_FFR
260 UNSPEC_UPDATE_FFRT
261 UNSPEC_RDFFR
262 UNSPEC_WRFFR
263 ;; Represents an SVE-style lane index, in which the indexing applies
264 ;; within the containing 128-bit block.
265 UNSPEC_SVE_LANE_SELECT
266 UNSPEC_SVE_CNT_PAT
267 UNSPEC_SVE_PREFETCH
268 UNSPEC_SVE_PREFETCH_GATHER
269 UNSPEC_SVE_COMPACT
270 UNSPEC_SVE_SPLICE
271 ])
272
273 (define_c_enum "unspecv" [
274 UNSPECV_EH_RETURN ; Represent EH_RETURN
275 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
276 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
277 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
278 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
279 UNSPECV_BLOCKAGE ; Represent a blockage
280 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
281 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
282 UNSPECV_BTI_NOARG ; Represent BTI.
283 UNSPECV_BTI_C ; Represent BTI c.
284 UNSPECV_BTI_J ; Represent BTI j.
285 UNSPECV_BTI_JC ; Represent BTI jc.
286 UNSPECV_TSTART ; Represent transaction start.
287 UNSPECV_TCOMMIT ; Represent transaction commit.
288 UNSPECV_TCANCEL ; Represent transaction cancel.
289 UNSPEC_RNDR ; Represent RNDR
290 UNSPEC_RNDRRS ; Represent RNDRRS
291 ]
292 )
293
294 ;; These constants are used as a const_int in various SVE unspecs
295 ;; to indicate whether the governing predicate is known to be a PTRUE.
296 (define_constants
297 [; Indicates that the predicate might not be a PTRUE.
298 (SVE_MAYBE_NOT_PTRUE 0)
299
300 ; Indicates that the predicate is known to be a PTRUE.
301 (SVE_KNOWN_PTRUE 1)])
302
303 ;; These constants are used as a const_int in predicated SVE FP arithmetic
304 ;; to indicate whether the operation is allowed to make additional lanes
305 ;; active without worrying about the effect on faulting behavior.
306 (define_constants
307 [; Indicates either that all lanes are active or that the instruction may
308 ; operate on inactive inputs even if doing so could induce a fault.
309 (SVE_RELAXED_GP 0)
310
311 ; Indicates that some lanes might be inactive and that the instruction
312 ; must not operate on inactive inputs if doing so could induce a fault.
313 (SVE_STRICT_GP 1)])
314
315 ;; If further include files are added the defintion of MD_INCLUDES
316 ;; must be updated.
317
318 (include "constraints.md")
319 (include "predicates.md")
320 (include "iterators.md")
321
322 ;; -------------------------------------------------------------------
323 ;; Instruction types and attributes
324 ;; -------------------------------------------------------------------
325
326 ; The "type" attribute is included here from AArch32 backend to be able
327 ; to share pipeline descriptions.
328 (include "../arm/types.md")
329
330 ;; It is important to set the fp or simd attributes to yes when a pattern
331 ;; alternative uses the FP or SIMD register files, usually signified by use of
332 ;; the 'w' constraint. This will ensure that the alternative will be
333 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
334 ;; architecture extensions. If all the alternatives in a pattern use the
335 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
336 ;; or TARGET_SIMD.
337
338 ;; Attributes of the architecture required to support the instruction (or
339 ;; alternative). This attribute is used to compute attribute "enabled", use type
340 ;; "any" to enable an alternative in all cases.
341
342 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
343
344 (define_enum_attr "arch" "arches" (const_string "any"))
345
346 ;; [For compatibility with Arm in pipeline models]
347 ;; Attribute that specifies whether or not the instruction touches fp
348 ;; registers.
349 ;; Note that this attribute is not used anywhere in either the arm or aarch64
350 ;; backends except in the scheduling description for xgene1. In that
351 ;; scheduling description this attribute is used to subclass the load_4 and
352 ;; load_8 types.
353 (define_attr "fp" "no,yes"
354 (if_then_else
355 (eq_attr "arch" "fp")
356 (const_string "yes")
357 (const_string "no")))
358
359 (define_attr "arch_enabled" "no,yes"
360 (if_then_else
361 (ior
362 (eq_attr "arch" "any")
363
364 (and (eq_attr "arch" "rcpc8_4")
365 (match_test "AARCH64_ISA_RCPC8_4"))
366
367 (and (eq_attr "arch" "fp")
368 (match_test "TARGET_FLOAT"))
369
370 (and (eq_attr "arch" "simd")
371 (match_test "TARGET_SIMD"))
372
373 (and (eq_attr "arch" "fp16")
374 (match_test "TARGET_FP_F16INST"))
375
376 (and (eq_attr "arch" "sve")
377 (match_test "TARGET_SVE")))
378 (const_string "yes")
379 (const_string "no")))
380
381 ;; Attribute that controls whether an alternative is enabled or not.
382 ;; Currently it is only used to disable alternatives which touch fp or simd
383 ;; registers when -mgeneral-regs-only is specified or to require a special
384 ;; architecture support.
385 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
386
387 ;; Attribute that specifies whether we are dealing with a branch to a
388 ;; label that is far away, i.e. further away than the maximum/minimum
389 ;; representable in a signed 21-bits number.
390 ;; 0 :=: no
391 ;; 1 :=: yes
392 (define_attr "far_branch" "" (const_int 0))
393
394 ;; Attribute that specifies whether the alternative uses MOVPRFX.
395 (define_attr "movprfx" "no,yes" (const_string "no"))
396
397 (define_attr "length" ""
398 (cond [(eq_attr "movprfx" "yes")
399 (const_int 8)
400 ] (const_int 4)))
401
402 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
403 ;; no predicated insns.
404 (define_attr "predicated" "yes,no" (const_string "no"))
405
406 ;; Set to true on an insn that requires the speculation tracking state to be
407 ;; in the tracking register before the insn issues. Otherwise the compiler
408 ;; may chose to hold the tracking state encoded in SP.
409 (define_attr "speculation_barrier" "true,false" (const_string "false"))
410
411 ;; -------------------------------------------------------------------
412 ;; Pipeline descriptions and scheduling
413 ;; -------------------------------------------------------------------
414
415 ;; Processor types.
416 (include "aarch64-tune.md")
417
418 ;; Scheduling
419 (include "../arm/cortex-a53.md")
420 (include "../arm/cortex-a57.md")
421 (include "../arm/exynos-m1.md")
422 (include "falkor.md")
423 (include "saphira.md")
424 (include "thunderx.md")
425 (include "../arm/xgene1.md")
426 (include "thunderx2t99.md")
427 (include "tsv110.md")
428
429 ;; -------------------------------------------------------------------
430 ;; Jumps and other miscellaneous insns
431 ;; -------------------------------------------------------------------
432
433 (define_insn "indirect_jump"
434 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
435 ""
436 "br\\t%0"
437 [(set_attr "type" "branch")]
438 )
439
440 (define_insn "jump"
441 [(set (pc) (label_ref (match_operand 0 "" "")))]
442 ""
443 "b\\t%l0"
444 [(set_attr "type" "branch")]
445 )
446
447 (define_expand "cbranch<mode>4"
448 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
449 [(match_operand:GPI 1 "register_operand")
450 (match_operand:GPI 2 "aarch64_plus_operand")])
451 (label_ref (match_operand 3 "" ""))
452 (pc)))]
453 ""
454 "
455 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
456 operands[2]);
457 operands[2] = const0_rtx;
458 "
459 )
460
461 (define_expand "cbranch<mode>4"
462 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
463 [(match_operand:GPF 1 "register_operand")
464 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
465 (label_ref (match_operand 3 "" ""))
466 (pc)))]
467 ""
468 "
469 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
470 operands[2]);
471 operands[2] = const0_rtx;
472 "
473 )
474
475 (define_expand "cbranchcc4"
476 [(set (pc) (if_then_else
477 (match_operator 0 "aarch64_comparison_operator"
478 [(match_operand 1 "cc_register")
479 (match_operand 2 "const0_operand")])
480 (label_ref (match_operand 3 "" ""))
481 (pc)))]
482 ""
483 "")
484
485 (define_insn "ccmp<mode>"
486 [(set (match_operand:CC 1 "cc_register" "")
487 (if_then_else:CC
488 (match_operator 4 "aarch64_comparison_operator"
489 [(match_operand 0 "cc_register" "")
490 (const_int 0)])
491 (compare:CC
492 (match_operand:GPI 2 "register_operand" "r,r,r")
493 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
494 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
495 ""
496 "@
497 ccmp\\t%<w>2, %<w>3, %k5, %m4
498 ccmp\\t%<w>2, %3, %k5, %m4
499 ccmn\\t%<w>2, #%n3, %k5, %m4"
500 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
501 )
502
503 (define_insn "fccmp<mode>"
504 [(set (match_operand:CCFP 1 "cc_register" "")
505 (if_then_else:CCFP
506 (match_operator 4 "aarch64_comparison_operator"
507 [(match_operand 0 "cc_register" "")
508 (const_int 0)])
509 (compare:CCFP
510 (match_operand:GPF 2 "register_operand" "w")
511 (match_operand:GPF 3 "register_operand" "w"))
512 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
513 "TARGET_FLOAT"
514 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
515 [(set_attr "type" "fccmp<s>")]
516 )
517
518 (define_insn "fccmpe<mode>"
519 [(set (match_operand:CCFPE 1 "cc_register" "")
520 (if_then_else:CCFPE
521 (match_operator 4 "aarch64_comparison_operator"
522 [(match_operand 0 "cc_register" "")
523 (const_int 0)])
524 (compare:CCFPE
525 (match_operand:GPF 2 "register_operand" "w")
526 (match_operand:GPF 3 "register_operand" "w"))
527 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
528 "TARGET_FLOAT"
529 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
530 [(set_attr "type" "fccmp<s>")]
531 )
532
533 ;; Expansion of signed mod by a power of 2 using CSNEG.
534 ;; For x0 % n where n is a power of 2 produce:
535 ;; negs x1, x0
536 ;; and x0, x0, #(n - 1)
537 ;; and x1, x1, #(n - 1)
538 ;; csneg x0, x0, x1, mi
539
540 (define_expand "mod<mode>3"
541 [(match_operand:GPI 0 "register_operand")
542 (match_operand:GPI 1 "register_operand")
543 (match_operand:GPI 2 "const_int_operand")]
544 ""
545 {
546 HOST_WIDE_INT val = INTVAL (operands[2]);
547
548 if (val <= 0
549 || exact_log2 (val) <= 0
550 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
551 FAIL;
552
553 rtx mask = GEN_INT (val - 1);
554
555 /* In the special case of x0 % 2 we can do the even shorter:
556 cmp x0, xzr
557 and x0, x0, 1
558 cneg x0, x0, lt. */
559 if (val == 2)
560 {
561 rtx masked = gen_reg_rtx (<MODE>mode);
562 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
563 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
564 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
565 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
566 DONE;
567 }
568
569 rtx neg_op = gen_reg_rtx (<MODE>mode);
570 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
571
572 /* Extract the condition register and mode. */
573 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
574 rtx cc_reg = SET_DEST (cmp);
575 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
576
577 rtx masked_pos = gen_reg_rtx (<MODE>mode);
578 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
579
580 rtx masked_neg = gen_reg_rtx (<MODE>mode);
581 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
582
583 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
584 masked_neg, masked_pos));
585 DONE;
586 }
587 )
588
589 (define_insn "condjump"
590 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
591 [(match_operand 1 "cc_register" "") (const_int 0)])
592 (label_ref (match_operand 2 "" ""))
593 (pc)))]
594 ""
595 {
596 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
597 but the "." is required for SVE conditions. */
598 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
599 if (get_attr_length (insn) == 8)
600 return aarch64_gen_far_branch (operands, 2, "Lbcond",
601 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
602 else
603 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
604 }
605 [(set_attr "type" "branch")
606 (set (attr "length")
607 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
608 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
609 (const_int 4)
610 (const_int 8)))
611 (set (attr "far_branch")
612 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
613 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
614 (const_int 0)
615 (const_int 1)))]
616 )
617
618 ;; For a 24-bit immediate CST we can optimize the compare for equality
619 ;; and branch sequence from:
620 ;; mov x0, #imm1
621 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
622 ;; cmp x1, x0
623 ;; b<ne,eq> .Label
624 ;; into the shorter:
625 ;; sub x0, x1, #(CST & 0xfff000)
626 ;; subs x0, x0, #(CST & 0x000fff)
627 ;; b<ne,eq> .Label
628 (define_insn_and_split "*compare_condjump<GPI:mode>"
629 [(set (pc) (if_then_else (EQL
630 (match_operand:GPI 0 "register_operand" "r")
631 (match_operand:GPI 1 "aarch64_imm24" "n"))
632 (label_ref:P (match_operand 2 "" ""))
633 (pc)))]
634 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
635 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
636 && !reload_completed"
637 "#"
638 "&& true"
639 [(const_int 0)]
640 {
641 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
642 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
643 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
644 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
645 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
646 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
647 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
648 cc_reg, const0_rtx);
649 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
650 DONE;
651 }
652 )
653
654 (define_expand "casesi"
655 [(match_operand:SI 0 "register_operand") ; Index
656 (match_operand:SI 1 "const_int_operand") ; Lower bound
657 (match_operand:SI 2 "const_int_operand") ; Total range
658 (match_operand:DI 3 "" "") ; Table label
659 (match_operand:DI 4 "" "")] ; Out of range label
660 ""
661 {
662 if (operands[1] != const0_rtx)
663 {
664 rtx reg = gen_reg_rtx (SImode);
665
666 /* Canonical RTL says that if you have:
667
668 (minus (X) (CONST))
669
670 then this should be emitted as:
671
672 (plus (X) (-CONST))
673
674 The use of trunc_int_for_mode ensures that the resulting
675 constant can be represented in SImode, this is important
676 for the corner case where operand[1] is INT_MIN. */
677
678 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
679
680 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
681 (operands[1], SImode))
682 operands[1] = force_reg (SImode, operands[1]);
683 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
684 operands[0] = reg;
685 }
686
687 if (!aarch64_plus_operand (operands[2], SImode))
688 operands[2] = force_reg (SImode, operands[2]);
689 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
690 const0_rtx),
691 operands[0], operands[2], operands[4]));
692
693 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
694 operands[2]
695 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
696 UNSPEC_CASESI);
697 operands[2] = gen_rtx_MEM (DImode, operands[2]);
698 MEM_READONLY_P (operands[2]) = 1;
699 MEM_NOTRAP_P (operands[2]) = 1;
700 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
701 DONE;
702 }
703 )
704
705 (define_expand "casesi_dispatch"
706 [(parallel
707 [(set (pc) (match_operand:DI 0 ""))
708 (clobber (reg:CC CC_REGNUM))
709 (clobber (match_scratch:DI 2))
710 (clobber (match_scratch:DI 3))
711 (use (label_ref:DI (match_operand 1 "")))])]
712 "")
713
714 (define_insn "*casesi_dispatch"
715 [(parallel
716 [(set (pc)
717 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
718 (match_operand:SI 1 "register_operand" "r")]
719 UNSPEC_CASESI)))
720 (clobber (reg:CC CC_REGNUM))
721 (clobber (match_scratch:DI 3 "=r"))
722 (clobber (match_scratch:DI 4 "=r"))
723 (use (label_ref:DI (match_operand 2 "" "")))])]
724 ""
725 "*
726 return aarch64_output_casesi (operands);
727 "
728 [(set_attr "length" "16")
729 (set_attr "type" "branch")]
730 )
731
732 (define_insn "nop"
733 [(unspec[(const_int 0)] UNSPEC_NOP)]
734 ""
735 "nop"
736 [(set_attr "type" "no_insn")]
737 )
738
739 (define_insn "prefetch"
740 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
741 (match_operand:QI 1 "const_int_operand" "")
742 (match_operand:QI 2 "const_int_operand" ""))]
743 ""
744 {
745 const char * pftype[2][4] =
746 {
747 {"prfm\\tPLDL1STRM, %0",
748 "prfm\\tPLDL3KEEP, %0",
749 "prfm\\tPLDL2KEEP, %0",
750 "prfm\\tPLDL1KEEP, %0"},
751 {"prfm\\tPSTL1STRM, %0",
752 "prfm\\tPSTL3KEEP, %0",
753 "prfm\\tPSTL2KEEP, %0",
754 "prfm\\tPSTL1KEEP, %0"},
755 };
756
757 int locality = INTVAL (operands[2]);
758
759 gcc_assert (IN_RANGE (locality, 0, 3));
760
761 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
762 the address into a DImode MEM so that aarch64_print_operand knows
763 how to print it. */
764 operands[0] = gen_rtx_MEM (DImode, operands[0]);
765 return pftype[INTVAL(operands[1])][locality];
766 }
767 [(set_attr "type" "load_4")]
768 )
769
770 (define_insn "trap"
771 [(trap_if (const_int 1) (const_int 8))]
772 ""
773 "brk #1000"
774 [(set_attr "type" "trap")])
775
776 (define_expand "prologue"
777 [(clobber (const_int 0))]
778 ""
779 "
780 aarch64_expand_prologue ();
781 DONE;
782 "
783 )
784
785 (define_expand "epilogue"
786 [(clobber (const_int 0))]
787 ""
788 "
789 aarch64_expand_epilogue (false);
790 DONE;
791 "
792 )
793
794 (define_expand "sibcall_epilogue"
795 [(clobber (const_int 0))]
796 ""
797 "
798 aarch64_expand_epilogue (true);
799 DONE;
800 "
801 )
802
803 (define_insn "*do_return"
804 [(return)]
805 ""
806 {
807 if (aarch64_return_address_signing_enabled ()
808 && TARGET_ARMV8_3
809 && !crtl->calls_eh_return)
810 {
811 if (aarch64_ra_sign_key == AARCH64_KEY_B)
812 return "retab";
813 else
814 return "retaa";
815 }
816 return "ret";
817 }
818 [(set_attr "type" "branch")]
819 )
820
821 (define_expand "return"
822 [(simple_return)]
823 "aarch64_use_return_insn_p ()"
824 ""
825 )
826
827 (define_insn "simple_return"
828 [(simple_return)]
829 ""
830 "ret"
831 [(set_attr "type" "branch")]
832 )
833
834 (define_insn "*cb<optab><mode>1"
835 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
836 (const_int 0))
837 (label_ref (match_operand 1 "" ""))
838 (pc)))]
839 "!aarch64_track_speculation"
840 {
841 if (get_attr_length (insn) == 8)
842 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
843 else
844 return "<cbz>\\t%<w>0, %l1";
845 }
846 [(set_attr "type" "branch")
847 (set (attr "length")
848 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
849 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
850 (const_int 4)
851 (const_int 8)))
852 (set (attr "far_branch")
853 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
854 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
855 (const_int 0)
856 (const_int 1)))]
857 )
858
859 (define_insn "*tb<optab><mode>1"
860 [(set (pc) (if_then_else
861 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
862 (const_int 1)
863 (match_operand 1
864 "aarch64_simd_shift_imm_<mode>" "n"))
865 (const_int 0))
866 (label_ref (match_operand 2 "" ""))
867 (pc)))
868 (clobber (reg:CC CC_REGNUM))]
869 "!aarch64_track_speculation"
870 {
871 if (get_attr_length (insn) == 8)
872 {
873 if (get_attr_far_branch (insn) == 1)
874 return aarch64_gen_far_branch (operands, 2, "Ltb",
875 "<inv_tb>\\t%<w>0, %1, ");
876 else
877 {
878 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
879 return "tst\t%<w>0, %1\;<bcond>\t%l2";
880 }
881 }
882 else
883 return "<tbz>\t%<w>0, %1, %l2";
884 }
885 [(set_attr "type" "branch")
886 (set (attr "length")
887 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
888 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
889 (const_int 4)
890 (const_int 8)))
891 (set (attr "far_branch")
892 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
893 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
894 (const_int 0)
895 (const_int 1)))]
896
897 )
898
899 (define_insn "*cb<optab><mode>1"
900 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
901 (const_int 0))
902 (label_ref (match_operand 1 "" ""))
903 (pc)))
904 (clobber (reg:CC CC_REGNUM))]
905 "!aarch64_track_speculation"
906 {
907 if (get_attr_length (insn) == 8)
908 {
909 if (get_attr_far_branch (insn) == 1)
910 return aarch64_gen_far_branch (operands, 1, "Ltb",
911 "<inv_tb>\\t%<w>0, <sizem1>, ");
912 else
913 {
914 char buf[64];
915 uint64_t val = ((uint64_t) 1)
916 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
917 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
918 output_asm_insn (buf, operands);
919 return "<bcond>\t%l1";
920 }
921 }
922 else
923 return "<tbz>\t%<w>0, <sizem1>, %l1";
924 }
925 [(set_attr "type" "branch")
926 (set (attr "length")
927 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
928 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
929 (const_int 4)
930 (const_int 8)))
931 (set (attr "far_branch")
932 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
933 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
934 (const_int 0)
935 (const_int 1)))]
936 )
937
938 ;; -------------------------------------------------------------------
939 ;; Subroutine calls and sibcalls
940 ;; -------------------------------------------------------------------
941
942 (define_expand "call"
943 [(parallel
944 [(call (match_operand 0 "memory_operand")
945 (match_operand 1 "general_operand"))
946 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
947 (clobber (reg:DI LR_REGNUM))])]
948 ""
949 "
950 {
951 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
952 DONE;
953 }"
954 )
955
956 (define_insn "*call_insn"
957 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
958 (match_operand 1 "" ""))
959 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
960 (clobber (reg:DI LR_REGNUM))]
961 ""
962 "@
963 blr\\t%0
964 bl\\t%c0"
965 [(set_attr "type" "call, call")]
966 )
967
968 (define_expand "call_value"
969 [(parallel
970 [(set (match_operand 0 "")
971 (call (match_operand 1 "memory_operand")
972 (match_operand 2 "general_operand")))
973 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
974 (clobber (reg:DI LR_REGNUM))])]
975 ""
976 "
977 {
978 aarch64_expand_call (operands[0], operands[1], operands[3], false);
979 DONE;
980 }"
981 )
982
983 (define_insn "*call_value_insn"
984 [(set (match_operand 0 "" "")
985 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
986 (match_operand 2 "" "")))
987 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
988 (clobber (reg:DI LR_REGNUM))]
989 ""
990 "@
991 blr\\t%1
992 bl\\t%c1"
993 [(set_attr "type" "call, call")]
994 )
995
996 (define_expand "sibcall"
997 [(parallel
998 [(call (match_operand 0 "memory_operand")
999 (match_operand 1 "general_operand"))
1000 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1001 (return)])]
1002 ""
1003 {
1004 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
1005 DONE;
1006 }
1007 )
1008
1009 (define_expand "sibcall_value"
1010 [(parallel
1011 [(set (match_operand 0 "")
1012 (call (match_operand 1 "memory_operand")
1013 (match_operand 2 "general_operand")))
1014 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1015 (return)])]
1016 ""
1017 {
1018 aarch64_expand_call (operands[0], operands[1], operands[3], true);
1019 DONE;
1020 }
1021 )
1022
1023 (define_insn "*sibcall_insn"
1024 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
1025 (match_operand 1 ""))
1026 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1027 (return)]
1028 "SIBLING_CALL_P (insn)"
1029 "@
1030 br\\t%0
1031 b\\t%c0"
1032 [(set_attr "type" "branch, branch")]
1033 )
1034
1035 (define_insn "*sibcall_value_insn"
1036 [(set (match_operand 0 "")
1037 (call (mem:DI
1038 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1039 (match_operand 2 "")))
1040 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1041 (return)]
1042 "SIBLING_CALL_P (insn)"
1043 "@
1044 br\\t%1
1045 b\\t%c1"
1046 [(set_attr "type" "branch, branch")]
1047 )
1048
1049 ;; Call subroutine returning any type.
1050
1051 (define_expand "untyped_call"
1052 [(parallel [(call (match_operand 0 "")
1053 (const_int 0))
1054 (match_operand 1 "")
1055 (match_operand 2 "")])]
1056 ""
1057 {
1058 int i;
1059
1060 /* Untyped calls always use the default ABI. It's only possible to use
1061 ABI variants if we know the type of the target function. */
1062 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));
1063
1064 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1065 {
1066 rtx set = XVECEXP (operands[2], 0, i);
1067 emit_move_insn (SET_DEST (set), SET_SRC (set));
1068 }
1069
1070 /* The optimizer does not know that the call sets the function value
1071 registers we stored in the result block. We avoid problems by
1072 claiming that all hard registers are used and clobbered at this
1073 point. */
1074 emit_insn (gen_blockage ());
1075 DONE;
1076 })
1077
1078 ;; -------------------------------------------------------------------
1079 ;; Moves
1080 ;; -------------------------------------------------------------------
1081
1082 (define_expand "mov<mode>"
1083 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1084 (match_operand:SHORT 1 "general_operand"))]
1085 ""
1086 "
1087 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1088 operands[1] = force_reg (<MODE>mode, operands[1]);
1089
1090 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1091 {
1092 aarch64_expand_mov_immediate (operands[0], operands[1]);
1093 DONE;
1094 }
1095 "
1096 )
1097
1098 (define_insn "*mov<mode>_aarch64"
1099 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
1100 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1101 "(register_operand (operands[0], <MODE>mode)
1102 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1103 {
1104 switch (which_alternative)
1105 {
1106 case 0:
1107 return "mov\t%w0, %w1";
1108 case 1:
1109 return "mov\t%w0, %1";
1110 case 2:
1111 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1112 <MODE>mode);
1113 case 3:
1114 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1115 case 4:
1116 return "ldr<size>\t%w0, %1";
1117 case 5:
1118 return "ldr\t%<size>0, %1";
1119 case 6:
1120 return "str<size>\t%w1, %0";
1121 case 7:
1122 return "str\t%<size>1, %0";
1123 case 8:
1124 return "umov\t%w0, %1.<v>[0]";
1125 case 9:
1126 return "dup\t%0.<Vallxd>, %w1";
1127 case 10:
1128 return "dup\t%<Vetype>0, %1.<v>[0]";
1129 default:
1130 gcc_unreachable ();
1131 }
1132 }
1133 ;; The "mov_imm" type for CNT is just a placeholder.
1134 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1135 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1136 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1137 )
1138
1139 (define_expand "mov<mode>"
1140 [(set (match_operand:GPI 0 "nonimmediate_operand")
1141 (match_operand:GPI 1 "general_operand"))]
1142 ""
1143 "
1144 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1145 && CONST_INT_P (operands[1]) && <MODE>mode == DImode
1146 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1147 DONE;
1148
1149 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1150 operands[1] = force_reg (<MODE>mode, operands[1]);
1151
1152 /* FIXME: RR we still need to fix up what we are doing with
1153 symbol_refs and other types of constants. */
1154 if (CONSTANT_P (operands[1])
1155 && !CONST_INT_P (operands[1]))
1156 {
1157 aarch64_expand_mov_immediate (operands[0], operands[1]);
1158 DONE;
1159 }
1160 "
1161 )
1162
1163 (define_insn_and_split "*movsi_aarch64"
1164 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
1165 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1166 "(register_operand (operands[0], SImode)
1167 || aarch64_reg_or_zero (operands[1], SImode))"
1168 "@
1169 mov\\t%w0, %w1
1170 mov\\t%w0, %w1
1171 mov\\t%w0, %w1
1172 mov\\t%w0, %1
1173 #
1174 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1175 ldr\\t%w0, %1
1176 ldr\\t%s0, %1
1177 str\\t%w1, %0
1178 str\\t%s1, %0
1179 adr\\t%x0, %c1
1180 adrp\\t%x0, %A1
1181 fmov\\t%s0, %w1
1182 fmov\\t%w0, %s1
1183 fmov\\t%s0, %s1
1184 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1185 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1186 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1187 [(const_int 0)]
1188 "{
1189 aarch64_expand_mov_immediate (operands[0], operands[1]);
1190 DONE;
1191 }"
1192 ;; The "mov_imm" type for CNT is just a placeholder.
1193 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1194 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1195 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1196 )
1197
1198 (define_insn_and_split "*movdi_aarch64"
1199 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1200 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1201 "(register_operand (operands[0], DImode)
1202 || aarch64_reg_or_zero (operands[1], DImode))"
1203 "@
1204 mov\\t%x0, %x1
1205 mov\\t%0, %x1
1206 mov\\t%x0, %1
1207 mov\\t%x0, %1
1208 mov\\t%w0, %1
1209 #
1210 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1211 ldr\\t%x0, %1
1212 ldr\\t%d0, %1
1213 str\\t%x1, %0
1214 str\\t%d1, %0
1215 adr\\t%x0, %c1
1216 adrp\\t%x0, %A1
1217 fmov\\t%d0, %x1
1218 fmov\\t%x0, %d1
1219 fmov\\t%d0, %d1
1220 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1221 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1222 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1223 [(const_int 0)]
1224 "{
1225 aarch64_expand_mov_immediate (operands[0], operands[1]);
1226 DONE;
1227 }"
1228 ;; The "mov_imm" type for CNTD is just a placeholder.
1229 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1230 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1231 neon_move")
1232 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1233 )
1234
1235 (define_insn "insv_imm<mode>"
1236 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1237 (const_int 16)
1238 (match_operand:GPI 1 "const_int_operand" "n"))
1239 (match_operand:GPI 2 "const_int_operand" "n"))]
1240 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1241 && UINTVAL (operands[1]) % 16 == 0"
1242 "movk\\t%<w>0, %X2, lsl %1"
1243 [(set_attr "type" "mov_imm")]
1244 )
1245
1246 (define_expand "movti"
1247 [(set (match_operand:TI 0 "nonimmediate_operand")
1248 (match_operand:TI 1 "general_operand"))]
1249 ""
1250 "
1251 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1252 operands[1] = force_reg (TImode, operands[1]);
1253
1254 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1255 {
1256 emit_move_insn (gen_lowpart (DImode, operands[0]),
1257 gen_lowpart (DImode, operands[1]));
1258 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1259 DONE;
1260 }
1261 "
1262 )
1263
1264 (define_insn "*movti_aarch64"
1265 [(set (match_operand:TI 0
1266 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
1267 (match_operand:TI 1
1268 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1269 "(register_operand (operands[0], TImode)
1270 || aarch64_reg_or_zero (operands[1], TImode))"
1271 "@
1272 #
1273 #
1274 #
1275 mov\\t%0.16b, %1.16b
1276 ldp\\t%0, %H0, %1
1277 stp\\t%1, %H1, %0
1278 stp\\txzr, xzr, %0
1279 ldr\\t%q0, %1
1280 str\\t%q1, %0"
1281 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1282 load_16,store_16,store_16,\
1283 load_16,store_16")
1284 (set_attr "length" "8,8,8,4,4,4,4,4,4")
1285 (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1286 )
1287
1288 ;; Split a TImode register-register or register-immediate move into
1289 ;; its component DImode pieces, taking care to handle overlapping
1290 ;; source and dest registers.
1291 (define_split
1292 [(set (match_operand:TI 0 "register_operand" "")
1293 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1294 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1295 [(const_int 0)]
1296 {
1297 aarch64_split_128bit_move (operands[0], operands[1]);
1298 DONE;
1299 })
1300
1301 (define_expand "mov<mode>"
1302 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1303 (match_operand:GPF_TF_F16 1 "general_operand"))]
1304 ""
1305 {
1306 if (!TARGET_FLOAT)
1307 {
1308 aarch64_err_no_fpadvsimd (<MODE>mode);
1309 FAIL;
1310 }
1311
1312 if (GET_CODE (operands[0]) == MEM
1313 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1314 && aarch64_float_const_zero_rtx_p (operands[1])))
1315 operands[1] = force_reg (<MODE>mode, operands[1]);
1316 }
1317 )
1318
1319 (define_insn "*movhf_aarch64"
1320 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1321 (match_operand:HF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1322 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1323 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1324 "@
1325 movi\\t%0.4h, #0
1326 fmov\\t%h0, %w1
1327 dup\\t%w0.4h, %w1
1328 umov\\t%w0, %1.h[0]
1329 mov\\t%0.h[0], %1.h[0]
1330 fmov\\t%h0, %1
1331 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1332 ldr\\t%h0, %1
1333 str\\t%h1, %0
1334 ldrh\\t%w0, %1
1335 strh\\t%w1, %0
1336 mov\\t%w0, %w1"
1337 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1338 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1339 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1340 )
1341
1342 (define_insn "*movsf_aarch64"
1343 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1344 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1345 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1346 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1347 "@
1348 movi\\t%0.2s, #0
1349 fmov\\t%s0, %w1
1350 fmov\\t%w0, %s1
1351 fmov\\t%s0, %s1
1352 fmov\\t%s0, %1
1353 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1354 ldr\\t%s0, %1
1355 str\\t%s1, %0
1356 ldr\\t%w0, %1
1357 str\\t%w1, %0
1358 mov\\t%w0, %w1
1359 mov\\t%w0, %1"
1360 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1361 f_loads,f_stores,load_4,store_4,mov_reg,\
1362 fconsts")
1363 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1364 )
1365
1366 (define_insn "*movdf_aarch64"
1367 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1368 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1369 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1370 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1371 "@
1372 movi\\t%d0, #0
1373 fmov\\t%d0, %x1
1374 fmov\\t%x0, %d1
1375 fmov\\t%d0, %d1
1376 fmov\\t%d0, %1
1377 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1378 ldr\\t%d0, %1
1379 str\\t%d1, %0
1380 ldr\\t%x0, %1
1381 str\\t%x1, %0
1382 mov\\t%x0, %x1
1383 mov\\t%x0, %1"
1384 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1385 f_loadd,f_stored,load_8,store_8,mov_reg,\
1386 fconstd")
1387 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1388 )
1389
1390 (define_split
1391 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1392 (match_operand:GPF_HF 1 "general_operand"))]
1393 "can_create_pseudo_p ()
1394 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1395 && !aarch64_float_const_representable_p (operands[1])
1396 && aarch64_float_const_rtx_p (operands[1])"
1397 [(const_int 0)]
1398 {
1399 unsigned HOST_WIDE_INT ival;
1400 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1401 FAIL;
1402
1403 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1404 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1405 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1406 DONE;
1407 }
1408 )
1409
1410 (define_insn "*movtf_aarch64"
1411 [(set (match_operand:TF 0
1412 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1413 (match_operand:TF 1
1414 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1415 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1416 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1417 "@
1418 mov\\t%0.16b, %1.16b
1419 #
1420 #
1421 #
1422 movi\\t%0.2d, #0
1423 fmov\\t%s0, wzr
1424 ldr\\t%q0, %1
1425 str\\t%q1, %0
1426 ldp\\t%0, %H0, %1
1427 stp\\t%1, %H1, %0
1428 stp\\txzr, xzr, %0"
1429 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1430 f_loadd,f_stored,load_16,store_16,store_16")
1431 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1432 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1433 )
1434
1435 (define_split
1436 [(set (match_operand:TF 0 "register_operand" "")
1437 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1438 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1439 [(const_int 0)]
1440 {
1441 aarch64_split_128bit_move (operands[0], operands[1]);
1442 DONE;
1443 }
1444 )
1445
1446 ;; 0 is dst
1447 ;; 1 is src
1448 ;; 2 is size of copy in bytes
1449 ;; 3 is alignment
1450
1451 (define_expand "cpymemdi"
1452 [(match_operand:BLK 0 "memory_operand")
1453 (match_operand:BLK 1 "memory_operand")
1454 (match_operand:DI 2 "immediate_operand")
1455 (match_operand:DI 3 "immediate_operand")]
1456 "!STRICT_ALIGNMENT"
1457 {
1458 if (aarch64_expand_cpymem (operands))
1459 DONE;
1460 FAIL;
1461 }
1462 )
1463
1464 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1465 ;; fairly lax checking on the second memory operation.
1466 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1467 [(set (match_operand:SX 0 "register_operand" "=r,w")
1468 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1469 (set (match_operand:SX2 2 "register_operand" "=r,w")
1470 (match_operand:SX2 3 "memory_operand" "m,m"))]
1471 "rtx_equal_p (XEXP (operands[3], 0),
1472 plus_constant (Pmode,
1473 XEXP (operands[1], 0),
1474 GET_MODE_SIZE (<SX:MODE>mode)))"
1475 "@
1476 ldp\\t%w0, %w2, %1
1477 ldp\\t%s0, %s2, %1"
1478 [(set_attr "type" "load_8,neon_load1_2reg")
1479 (set_attr "arch" "*,fp")]
1480 )
1481
1482 ;; Storing different modes that can still be merged
1483 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1484 [(set (match_operand:DX 0 "register_operand" "=r,w")
1485 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1486 (set (match_operand:DX2 2 "register_operand" "=r,w")
1487 (match_operand:DX2 3 "memory_operand" "m,m"))]
1488 "rtx_equal_p (XEXP (operands[3], 0),
1489 plus_constant (Pmode,
1490 XEXP (operands[1], 0),
1491 GET_MODE_SIZE (<DX:MODE>mode)))"
1492 "@
1493 ldp\\t%x0, %x2, %1
1494 ldp\\t%d0, %d2, %1"
1495 [(set_attr "type" "load_16,neon_load1_2reg")
1496 (set_attr "arch" "*,fp")]
1497 )
1498
1499 (define_insn "load_pair_dw_tftf"
1500 [(set (match_operand:TF 0 "register_operand" "=w")
1501 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1502 (set (match_operand:TF 2 "register_operand" "=w")
1503 (match_operand:TF 3 "memory_operand" "m"))]
1504 "TARGET_SIMD
1505 && rtx_equal_p (XEXP (operands[3], 0),
1506 plus_constant (Pmode,
1507 XEXP (operands[1], 0),
1508 GET_MODE_SIZE (TFmode)))"
1509 "ldp\\t%q0, %q2, %1"
1510 [(set_attr "type" "neon_ldp_q")
1511 (set_attr "fp" "yes")]
1512 )
1513
1514 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1515 ;; fairly lax checking on the second memory operation.
1516 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1517 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1518 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1519 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1520 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1521 "rtx_equal_p (XEXP (operands[2], 0),
1522 plus_constant (Pmode,
1523 XEXP (operands[0], 0),
1524 GET_MODE_SIZE (<SX:MODE>mode)))"
1525 "@
1526 stp\\t%w1, %w3, %0
1527 stp\\t%s1, %s3, %0"
1528 [(set_attr "type" "store_8,neon_store1_2reg")
1529 (set_attr "arch" "*,fp")]
1530 )
1531
1532 ;; Storing different modes that can still be merged
1533 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1534 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1535 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1536 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1537 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1538 "rtx_equal_p (XEXP (operands[2], 0),
1539 plus_constant (Pmode,
1540 XEXP (operands[0], 0),
1541 GET_MODE_SIZE (<DX:MODE>mode)))"
1542 "@
1543 stp\\t%x1, %x3, %0
1544 stp\\t%d1, %d3, %0"
1545 [(set_attr "type" "store_16,neon_store1_2reg")
1546 (set_attr "arch" "*,fp")]
1547 )
1548
1549 (define_insn "store_pair_dw_tftf"
1550 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1551 (match_operand:TF 1 "register_operand" "w"))
1552 (set (match_operand:TF 2 "memory_operand" "=m")
1553 (match_operand:TF 3 "register_operand" "w"))]
1554 "TARGET_SIMD &&
1555 rtx_equal_p (XEXP (operands[2], 0),
1556 plus_constant (Pmode,
1557 XEXP (operands[0], 0),
1558 GET_MODE_SIZE (TFmode)))"
1559 "stp\\t%q1, %q3, %0"
1560 [(set_attr "type" "neon_stp_q")
1561 (set_attr "fp" "yes")]
1562 )
1563
1564 ;; Load pair with post-index writeback. This is primarily used in function
1565 ;; epilogues.
1566 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1567 [(parallel
1568 [(set (match_operand:P 0 "register_operand" "=k")
1569 (plus:P (match_operand:P 1 "register_operand" "0")
1570 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1571 (set (match_operand:GPI 2 "register_operand" "=r")
1572 (mem:GPI (match_dup 1)))
1573 (set (match_operand:GPI 3 "register_operand" "=r")
1574 (mem:GPI (plus:P (match_dup 1)
1575 (match_operand:P 5 "const_int_operand" "n"))))])]
1576 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1577 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1578 [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1579 )
1580
1581 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1582 [(parallel
1583 [(set (match_operand:P 0 "register_operand" "=k")
1584 (plus:P (match_operand:P 1 "register_operand" "0")
1585 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1586 (set (match_operand:GPF 2 "register_operand" "=w")
1587 (mem:GPF (match_dup 1)))
1588 (set (match_operand:GPF 3 "register_operand" "=w")
1589 (mem:GPF (plus:P (match_dup 1)
1590 (match_operand:P 5 "const_int_operand" "n"))))])]
1591 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1592 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1593 [(set_attr "type" "neon_load1_2reg")]
1594 )
1595
1596 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1597 [(parallel
1598 [(set (match_operand:P 0 "register_operand" "=k")
1599 (plus:P (match_operand:P 1 "register_operand" "0")
1600 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1601 (set (match_operand:TX 2 "register_operand" "=w")
1602 (mem:TX (match_dup 1)))
1603 (set (match_operand:TX 3 "register_operand" "=w")
1604 (mem:TX (plus:P (match_dup 1)
1605 (match_operand:P 5 "const_int_operand" "n"))))])]
1606 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1607 "ldp\\t%q2, %q3, [%1], %4"
1608 [(set_attr "type" "neon_ldp_q")]
1609 )
1610
1611 ;; Store pair with pre-index writeback. This is primarily used in function
1612 ;; prologues.
1613 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1614 [(parallel
1615 [(set (match_operand:P 0 "register_operand" "=&k")
1616 (plus:P (match_operand:P 1 "register_operand" "0")
1617 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1618 (set (mem:GPI (plus:P (match_dup 0)
1619 (match_dup 4)))
1620 (match_operand:GPI 2 "register_operand" "r"))
1621 (set (mem:GPI (plus:P (match_dup 0)
1622 (match_operand:P 5 "const_int_operand" "n")))
1623 (match_operand:GPI 3 "register_operand" "r"))])]
1624 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1625 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1626 [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1627 )
1628
1629 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1630 [(parallel
1631 [(set (match_operand:P 0 "register_operand" "=&k")
1632 (plus:P (match_operand:P 1 "register_operand" "0")
1633 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1634 (set (mem:GPF (plus:P (match_dup 0)
1635 (match_dup 4)))
1636 (match_operand:GPF 2 "register_operand" "w"))
1637 (set (mem:GPF (plus:P (match_dup 0)
1638 (match_operand:P 5 "const_int_operand" "n")))
1639 (match_operand:GPF 3 "register_operand" "w"))])]
1640 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1641 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1642 [(set_attr "type" "neon_store1_2reg<q>")]
1643 )
1644
1645 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1646 [(parallel
1647 [(set (match_operand:P 0 "register_operand" "=&k")
1648 (plus:P (match_operand:P 1 "register_operand" "0")
1649 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1650 (set (mem:TX (plus:P (match_dup 0)
1651 (match_dup 4)))
1652 (match_operand:TX 2 "register_operand" "w"))
1653 (set (mem:TX (plus:P (match_dup 0)
1654 (match_operand:P 5 "const_int_operand" "n")))
1655 (match_operand:TX 3 "register_operand" "w"))])]
1656 "TARGET_SIMD
1657 && INTVAL (operands[5])
1658 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1659 "stp\\t%q2, %q3, [%0, %4]!"
1660 [(set_attr "type" "neon_stp_q")]
1661 )
1662
1663 ;; -------------------------------------------------------------------
1664 ;; Sign/Zero extension
1665 ;; -------------------------------------------------------------------
1666
1667 (define_expand "<optab>sidi2"
1668 [(set (match_operand:DI 0 "register_operand")
1669 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1670 ""
1671 )
1672
1673 (define_insn "*extendsidi2_aarch64"
1674 [(set (match_operand:DI 0 "register_operand" "=r,r")
1675 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1676 ""
1677 "@
1678 sxtw\t%0, %w1
1679 ldrsw\t%0, %1"
1680 [(set_attr "type" "extend,load_4")]
1681 )
1682
1683 (define_insn "*load_pair_extendsidi2_aarch64"
1684 [(set (match_operand:DI 0 "register_operand" "=r")
1685 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1686 (set (match_operand:DI 2 "register_operand" "=r")
1687 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1688 "rtx_equal_p (XEXP (operands[3], 0),
1689 plus_constant (Pmode,
1690 XEXP (operands[1], 0),
1691 GET_MODE_SIZE (SImode)))"
1692 "ldpsw\\t%0, %2, %1"
1693 [(set_attr "type" "load_8")]
1694 )
1695
1696 (define_insn "*zero_extendsidi2_aarch64"
1697 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1698 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1699 ""
1700 "@
1701 uxtw\t%0, %w1
1702 ldr\t%w0, %1
1703 fmov\t%s0, %w1
1704 ldr\t%s0, %1
1705 fmov\t%w0, %s1
1706 fmov\t%s0, %s1"
1707 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1708 (set_attr "arch" "*,*,fp,fp,fp,fp")]
1709 )
1710
1711 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1712 [(set (match_operand:DI 0 "register_operand" "=r,w")
1713 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1714 (set (match_operand:DI 2 "register_operand" "=r,w")
1715 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1716 "rtx_equal_p (XEXP (operands[3], 0),
1717 plus_constant (Pmode,
1718 XEXP (operands[1], 0),
1719 GET_MODE_SIZE (SImode)))"
1720 "@
1721 ldp\t%w0, %w2, %1
1722 ldp\t%s0, %s2, %1"
1723 [(set_attr "type" "load_8,neon_load1_2reg")
1724 (set_attr "arch" "*,fp")]
1725 )
1726
1727 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1728 [(set (match_operand:GPI 0 "register_operand")
1729 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1730 ""
1731 )
1732
1733 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1734 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1735 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1736 ""
1737 "@
1738 sxt<SHORT:size>\t%<GPI:w>0, %w1
1739 ldrs<SHORT:size>\t%<GPI:w>0, %1"
1740 [(set_attr "type" "extend,load_4")]
1741 )
1742
1743 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1744 [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1745 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1746 ""
1747 "@
1748 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1749 ldr<SHORT:size>\t%w0, %1
1750 ldr\t%<SHORT:size>0, %1"
1751 [(set_attr "type" "logic_imm,load_4,f_loads")
1752 (set_attr "arch" "*,*,fp")]
1753 )
1754
1755 (define_expand "<optab>qihi2"
1756 [(set (match_operand:HI 0 "register_operand")
1757 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1758 ""
1759 )
1760
1761 (define_insn "*extendqihi2_aarch64"
1762 [(set (match_operand:HI 0 "register_operand" "=r,r")
1763 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1764 ""
1765 "@
1766 sxtb\t%w0, %w1
1767 ldrsb\t%w0, %1"
1768 [(set_attr "type" "extend,load_4")]
1769 )
1770
1771 (define_insn "*zero_extendqihi2_aarch64"
1772 [(set (match_operand:HI 0 "register_operand" "=r,r")
1773 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1774 ""
1775 "@
1776 and\t%w0, %w1, 255
1777 ldrb\t%w0, %1"
1778 [(set_attr "type" "logic_imm,load_4")]
1779 )
1780
1781 ;; -------------------------------------------------------------------
1782 ;; Simple arithmetic
1783 ;; -------------------------------------------------------------------
1784
1785 (define_expand "add<mode>3"
1786 [(set
1787 (match_operand:GPI 0 "register_operand")
1788 (plus:GPI (match_operand:GPI 1 "register_operand")
1789 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1790 ""
1791 {
1792 /* If operands[1] is a subreg extract the inner RTX. */
1793 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1794
1795 /* If the constant is too large for a single instruction and isn't frame
1796 based, split off the immediate so it is available for CSE. */
1797 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1798 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
1799 && can_create_pseudo_p ()
1800 && (!REG_P (op1)
1801 || !REGNO_PTR_FRAME_P (REGNO (op1))))
1802 operands[2] = force_reg (<MODE>mode, operands[2]);
1803 /* Expand polynomial additions now if the destination is the stack
1804 pointer, since we don't want to use that as a temporary. */
1805 else if (operands[0] == stack_pointer_rtx
1806 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1807 {
1808 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1809 operands[2], NULL_RTX, NULL_RTX);
1810 DONE;
1811 }
1812 })
1813
1814 (define_insn "*add<mode>3_aarch64"
1815 [(set
1816 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
1817 (plus:GPI
1818 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1819 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
1820 ""
1821 "@
1822 add\\t%<w>0, %<w>1, %2
1823 add\\t%<w>0, %<w>1, %<w>2
1824 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1825 sub\\t%<w>0, %<w>1, #%n2
1826 #
1827 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1828 * return aarch64_output_sve_addvl_addpl (operands[2]);"
1829 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1830 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1831 (set_attr "arch" "*,*,simd,*,*,sve,sve")]
1832 )
1833
1834 ;; zero_extend version of above
1835 (define_insn "*addsi3_aarch64_uxtw"
1836 [(set
1837 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1838 (zero_extend:DI
1839 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1840 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1841 ""
1842 "@
1843 add\\t%w0, %w1, %2
1844 add\\t%w0, %w1, %w2
1845 sub\\t%w0, %w1, #%n2
1846 #"
1847 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1848 )
1849
1850 ;; If there's a free register, and we can load the constant with a
1851 ;; single instruction, do so. This has a chance to improve scheduling.
1852 (define_peephole2
1853 [(match_scratch:GPI 3 "r")
1854 (set (match_operand:GPI 0 "register_operand")
1855 (plus:GPI
1856 (match_operand:GPI 1 "register_operand")
1857 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1858 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1859 [(set (match_dup 3) (match_dup 2))
1860 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1861 )
1862
1863 (define_peephole2
1864 [(match_scratch:SI 3 "r")
1865 (set (match_operand:DI 0 "register_operand")
1866 (zero_extend:DI
1867 (plus:SI
1868 (match_operand:SI 1 "register_operand")
1869 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1870 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1871 [(set (match_dup 3) (match_dup 2))
1872 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1873 )
1874
1875 ;; After peephole2 has had a chance to run, split any remaining long
1876 ;; additions into two add immediates.
1877 (define_split
1878 [(set (match_operand:GPI 0 "register_operand")
1879 (plus:GPI
1880 (match_operand:GPI 1 "register_operand")
1881 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1882 "epilogue_completed"
1883 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1884 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1885 {
1886 HOST_WIDE_INT i = INTVAL (operands[2]);
1887 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1888 operands[3] = GEN_INT (i - s);
1889 operands[4] = GEN_INT (s);
1890 }
1891 )
1892
1893 ;; Match addition of polynomial offsets that require one temporary, for which
1894 ;; we can use the early-clobbered destination register. This is a separate
1895 ;; pattern so that the early clobber doesn't affect register allocation
1896 ;; for other forms of addition. However, we still need to provide an
1897 ;; all-register alternative, in case the offset goes out of range after
1898 ;; elimination. For completeness we might as well provide all GPR-based
1899 ;; alternatives from the main pattern.
1900 ;;
1901 ;; We don't have a pattern for additions requiring two temporaries since at
1902 ;; present LRA doesn't allow new scratches to be added during elimination.
1903 ;; Such offsets should be rare anyway.
1904 ;;
1905 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1906 ;; here would go away. We could just handle all polynomial constants in
1907 ;; this pattern.
1908 (define_insn_and_split "*add<mode>3_poly_1"
1909 [(set
1910 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
1911 (plus:GPI
1912 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
1913 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
1914 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1915 "@
1916 add\\t%<w>0, %<w>1, %2
1917 add\\t%<w>0, %<w>1, %<w>2
1918 sub\\t%<w>0, %<w>1, #%n2
1919 #
1920 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1921 * return aarch64_output_sve_addvl_addpl (operands[2]);
1922 #"
1923 "&& epilogue_completed
1924 && !reg_overlap_mentioned_p (operands[0], operands[1])
1925 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1926 [(const_int 0)]
1927 {
1928 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1929 operands[2], operands[0], NULL_RTX);
1930 DONE;
1931 }
1932 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1933 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
1934 )
1935
1936 (define_split
1937 [(set (match_operand:DI 0 "register_operand")
1938 (zero_extend:DI
1939 (plus:SI
1940 (match_operand:SI 1 "register_operand")
1941 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1942 "epilogue_completed"
1943 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1944 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1945 {
1946 HOST_WIDE_INT i = INTVAL (operands[2]);
1947 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1948 operands[3] = GEN_INT (i - s);
1949 operands[4] = GEN_INT (s);
1950 operands[5] = gen_lowpart (SImode, operands[0]);
1951 }
1952 )
1953
1954 (define_expand "addv<mode>4"
1955 [(match_operand:GPI 0 "register_operand")
1956 (match_operand:GPI 1 "register_operand")
1957 (match_operand:GPI 2 "aarch64_plus_operand")
1958 (label_ref (match_operand 3 "" ""))]
1959 ""
1960 {
1961 if (CONST_INT_P (operands[2]))
1962 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1963 operands[2]));
1964 else
1965 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1966 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1967
1968 DONE;
1969 })
1970
1971 (define_expand "uaddv<mode>4"
1972 [(match_operand:GPI 0 "register_operand")
1973 (match_operand:GPI 1 "register_operand")
1974 (match_operand:GPI 2 "register_operand")
1975 (label_ref (match_operand 3 "" ""))]
1976 ""
1977 {
1978 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1979 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1980
1981 DONE;
1982 })
1983
1984 (define_expand "addti3"
1985 [(set (match_operand:TI 0 "register_operand")
1986 (plus:TI (match_operand:TI 1 "register_operand")
1987 (match_operand:TI 2 "aarch64_reg_or_imm")))]
1988 ""
1989 {
1990 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1991
1992 aarch64_addti_scratch_regs (operands[1], operands[2],
1993 &low_dest, &op1_low, &op2_low,
1994 &high_dest, &op1_high, &op2_high);
1995
1996 if (op2_low == const0_rtx)
1997 {
1998 low_dest = op1_low;
1999 if (!aarch64_pluslong_operand (op2_high, DImode))
2000 op2_high = force_reg (DImode, op2_high);
2001 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
2002 }
2003 else
2004 {
2005 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2006 force_reg (DImode, op2_low)));
2007 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
2008 force_reg (DImode, op2_high)));
2009 }
2010
2011 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2012 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2013
2014 DONE;
2015 })
2016
2017 (define_expand "addvti4"
2018 [(match_operand:TI 0 "register_operand")
2019 (match_operand:TI 1 "register_operand")
2020 (match_operand:TI 2 "aarch64_reg_or_imm")
2021 (label_ref (match_operand 3 "" ""))]
2022 ""
2023 {
2024 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2025
2026 aarch64_addti_scratch_regs (operands[1], operands[2],
2027 &low_dest, &op1_low, &op2_low,
2028 &high_dest, &op1_high, &op2_high);
2029
2030 if (op2_low == const0_rtx)
2031 {
2032 low_dest = op1_low;
2033 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2034 force_reg (DImode, op2_high)));
2035 }
2036 else
2037 {
2038 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2039 force_reg (DImode, op2_low)));
2040 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2041 force_reg (DImode, op2_high)));
2042 }
2043
2044 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2045 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2046
2047 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2048 DONE;
2049 })
2050
2051 (define_expand "uaddvti4"
2052 [(match_operand:TI 0 "register_operand")
2053 (match_operand:TI 1 "register_operand")
2054 (match_operand:TI 2 "aarch64_reg_or_imm")
2055 (label_ref (match_operand 3 "" ""))]
2056 ""
2057 {
2058 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2059
2060 aarch64_addti_scratch_regs (operands[1], operands[2],
2061 &low_dest, &op1_low, &op2_low,
2062 &high_dest, &op1_high, &op2_high);
2063
2064 if (op2_low == const0_rtx)
2065 {
2066 low_dest = op1_low;
2067 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2068 force_reg (DImode, op2_high)));
2069 }
2070 else
2071 {
2072 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2073 force_reg (DImode, op2_low)));
2074 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2075 force_reg (DImode, op2_high)));
2076 }
2077
2078 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2079 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2080
2081 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2082 DONE;
2083 })
2084
2085 (define_insn "add<mode>3_compare0"
2086 [(set (reg:CC_NZ CC_REGNUM)
2087 (compare:CC_NZ
2088 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2089 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2090 (const_int 0)))
2091 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2092 (plus:GPI (match_dup 1) (match_dup 2)))]
2093 ""
2094 "@
2095 adds\\t%<w>0, %<w>1, %<w>2
2096 adds\\t%<w>0, %<w>1, %2
2097 subs\\t%<w>0, %<w>1, #%n2"
2098 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2099 )
2100
2101 ;; zero_extend version of above
2102 (define_insn "*addsi3_compare0_uxtw"
2103 [(set (reg:CC_NZ CC_REGNUM)
2104 (compare:CC_NZ
2105 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2106 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2107 (const_int 0)))
2108 (set (match_operand:DI 0 "register_operand" "=r,r,r")
2109 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2110 ""
2111 "@
2112 adds\\t%w0, %w1, %w2
2113 adds\\t%w0, %w1, %2
2114 subs\\t%w0, %w1, #%n2"
2115 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2116 )
2117
2118 (define_insn "*add<mode>3_compareC_cconly"
2119 [(set (reg:CC_C CC_REGNUM)
2120 (compare:CC_C
2121 (plus:GPI
2122 (match_operand:GPI 0 "register_operand" "r,r,r")
2123 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2124 (match_dup 0)))]
2125 ""
2126 "@
2127 cmn\\t%<w>0, %<w>1
2128 cmn\\t%<w>0, %1
2129 cmp\\t%<w>0, #%n1"
2130 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2131 )
2132
2133 (define_insn "add<mode>3_compareC"
2134 [(set (reg:CC_C CC_REGNUM)
2135 (compare:CC_C
2136 (plus:GPI
2137 (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2138 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2139 (match_dup 1)))
2140 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2141 (plus:GPI (match_dup 1) (match_dup 2)))]
2142 ""
2143 "@
2144 adds\\t%<w>0, %<w>1, %<w>2
2145 adds\\t%<w>0, %<w>1, %2
2146 subs\\t%<w>0, %<w>1, #%n2"
2147 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2148 )
2149
2150 (define_insn "*add<mode>3_compareV_cconly_imm"
2151 [(set (reg:CC_V CC_REGNUM)
2152 (compare:CC_V
2153 (plus:<DWI>
2154 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2155 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2156 (sign_extend:<DWI>
2157 (plus:GPI
2158 (match_dup 0)
2159 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2160 "INTVAL (operands[1]) == INTVAL (operands[2])"
2161 "@
2162 cmn\\t%<w>0, %<w>1
2163 cmp\\t%<w>0, #%n1"
2164 [(set_attr "type" "alus_imm")]
2165 )
2166
2167 (define_insn "*add<mode>3_compareV_cconly"
2168 [(set (reg:CC_V CC_REGNUM)
2169 (compare:CC_V
2170 (plus:<DWI>
2171 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2172 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2173 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2174 ""
2175 "cmn\\t%<w>0, %<w>1"
2176 [(set_attr "type" "alus_sreg")]
2177 )
2178
2179 (define_insn "add<mode>3_compareV_imm"
2180 [(set (reg:CC_V CC_REGNUM)
2181 (compare:CC_V
2182 (plus:<DWI>
2183 (sign_extend:<DWI>
2184 (match_operand:GPI 1 "register_operand" "rk,rk"))
2185 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2186 (sign_extend:<DWI>
2187 (plus:GPI (match_dup 1) (match_dup 2)))))
2188 (set (match_operand:GPI 0 "register_operand" "=r,r")
2189 (plus:GPI (match_dup 1) (match_dup 2)))]
2190 ""
2191 "@
2192 adds\\t%<w>0, %<w>1, %<w>2
2193 subs\\t%<w>0, %<w>1, #%n2"
2194 [(set_attr "type" "alus_imm,alus_imm")]
2195 )
2196
2197 (define_insn "add<mode>3_compareV"
2198 [(set (reg:CC_V CC_REGNUM)
2199 (compare:CC_V
2200 (plus:<DWI>
2201 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2202 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2203 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2204 (set (match_operand:GPI 0 "register_operand" "=r")
2205 (plus:GPI (match_dup 1) (match_dup 2)))]
2206 ""
2207 "adds\\t%<w>0, %<w>1, %<w>2"
2208 [(set_attr "type" "alus_sreg")]
2209 )
2210
2211 (define_insn "*adds_shift_imm_<mode>"
2212 [(set (reg:CC_NZ CC_REGNUM)
2213 (compare:CC_NZ
2214 (plus:GPI (ASHIFT:GPI
2215 (match_operand:GPI 1 "register_operand" "r")
2216 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2217 (match_operand:GPI 3 "register_operand" "r"))
2218 (const_int 0)))
2219 (set (match_operand:GPI 0 "register_operand" "=r")
2220 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2221 (match_dup 3)))]
2222 ""
2223 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2224 [(set_attr "type" "alus_shift_imm")]
2225 )
2226
2227 (define_insn "*subs_shift_imm_<mode>"
2228 [(set (reg:CC_NZ CC_REGNUM)
2229 (compare:CC_NZ
2230 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2231 (ASHIFT:GPI
2232 (match_operand:GPI 2 "register_operand" "r")
2233 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2234 (const_int 0)))
2235 (set (match_operand:GPI 0 "register_operand" "=r")
2236 (minus:GPI (match_dup 1)
2237 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2238 ""
2239 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2240 [(set_attr "type" "alus_shift_imm")]
2241 )
2242
2243 (define_insn "*adds_mul_imm_<mode>"
2244 [(set (reg:CC_NZ CC_REGNUM)
2245 (compare:CC_NZ
2246 (plus:GPI (mult:GPI
2247 (match_operand:GPI 1 "register_operand" "r")
2248 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2249 (match_operand:GPI 3 "register_operand" "r"))
2250 (const_int 0)))
2251 (set (match_operand:GPI 0 "register_operand" "=r")
2252 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2253 (match_dup 3)))]
2254 ""
2255 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2256 [(set_attr "type" "alus_shift_imm")]
2257 )
2258
2259 (define_insn "*subs_mul_imm_<mode>"
2260 [(set (reg:CC_NZ CC_REGNUM)
2261 (compare:CC_NZ
2262 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2263 (mult:GPI
2264 (match_operand:GPI 2 "register_operand" "r")
2265 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2266 (const_int 0)))
2267 (set (match_operand:GPI 0 "register_operand" "=r")
2268 (minus:GPI (match_dup 1)
2269 (mult:GPI (match_dup 2) (match_dup 3))))]
2270 ""
2271 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2272 [(set_attr "type" "alus_shift_imm")]
2273 )
2274
2275 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2276 [(set (reg:CC_NZ CC_REGNUM)
2277 (compare:CC_NZ
2278 (plus:GPI
2279 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2280 (match_operand:GPI 2 "register_operand" "rk"))
2281 (const_int 0)))
2282 (set (match_operand:GPI 0 "register_operand" "=r")
2283 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2284 ""
2285 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2286 [(set_attr "type" "alus_ext")]
2287 )
2288
2289 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2290 [(set (reg:CC_NZ CC_REGNUM)
2291 (compare:CC_NZ
2292 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2293 (ANY_EXTEND:GPI
2294 (match_operand:ALLX 2 "register_operand" "r")))
2295 (const_int 0)))
2296 (set (match_operand:GPI 0 "register_operand" "=r")
2297 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2298 ""
2299 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2300 [(set_attr "type" "alus_ext")]
2301 )
2302
2303 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2304 [(set (reg:CC_NZ CC_REGNUM)
2305 (compare:CC_NZ
2306 (plus:GPI (ashift:GPI
2307 (ANY_EXTEND:GPI
2308 (match_operand:ALLX 1 "register_operand" "r"))
2309 (match_operand 2 "aarch64_imm3" "Ui3"))
2310 (match_operand:GPI 3 "register_operand" "rk"))
2311 (const_int 0)))
2312 (set (match_operand:GPI 0 "register_operand" "=rk")
2313 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2314 (match_dup 2))
2315 (match_dup 3)))]
2316 ""
2317 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2318 [(set_attr "type" "alus_ext")]
2319 )
2320
2321 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2322 [(set (reg:CC_NZ CC_REGNUM)
2323 (compare:CC_NZ
2324 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2325 (ashift:GPI
2326 (ANY_EXTEND:GPI
2327 (match_operand:ALLX 2 "register_operand" "r"))
2328 (match_operand 3 "aarch64_imm3" "Ui3")))
2329 (const_int 0)))
2330 (set (match_operand:GPI 0 "register_operand" "=rk")
2331 (minus:GPI (match_dup 1)
2332 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2333 (match_dup 3))))]
2334 ""
2335 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2336 [(set_attr "type" "alus_ext")]
2337 )
2338
2339 (define_insn "*adds_<optab><mode>_multp2"
2340 [(set (reg:CC_NZ CC_REGNUM)
2341 (compare:CC_NZ
2342 (plus:GPI (ANY_EXTRACT:GPI
2343 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2344 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2345 (match_operand 3 "const_int_operand" "n")
2346 (const_int 0))
2347 (match_operand:GPI 4 "register_operand" "rk"))
2348 (const_int 0)))
2349 (set (match_operand:GPI 0 "register_operand" "=r")
2350 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2351 (match_dup 3)
2352 (const_int 0))
2353 (match_dup 4)))]
2354 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2355 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2356 [(set_attr "type" "alus_ext")]
2357 )
2358
2359 (define_insn "*subs_<optab><mode>_multp2"
2360 [(set (reg:CC_NZ CC_REGNUM)
2361 (compare:CC_NZ
2362 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2363 (ANY_EXTRACT:GPI
2364 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2365 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2366 (match_operand 3 "const_int_operand" "n")
2367 (const_int 0)))
2368 (const_int 0)))
2369 (set (match_operand:GPI 0 "register_operand" "=r")
2370 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2371 (mult:GPI (match_dup 1) (match_dup 2))
2372 (match_dup 3)
2373 (const_int 0))))]
2374 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2375 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2376 [(set_attr "type" "alus_ext")]
2377 )
2378
2379 (define_insn "*add<mode>3nr_compare0"
2380 [(set (reg:CC_NZ CC_REGNUM)
2381 (compare:CC_NZ
2382 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2383 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2384 (const_int 0)))]
2385 ""
2386 "@
2387 cmn\\t%<w>0, %<w>1
2388 cmn\\t%<w>0, %1
2389 cmp\\t%<w>0, #%n1"
2390 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2391 )
2392
2393 (define_insn "aarch64_sub<mode>_compare0"
2394 [(set (reg:CC_NZ CC_REGNUM)
2395 (compare:CC_NZ
2396 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2397 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2398 (const_int 0)))]
2399 ""
2400 "cmp\\t%<w>0, %<w>1"
2401 [(set_attr "type" "alus_sreg")]
2402 )
2403
2404 (define_insn "*compare_neg<mode>"
2405 [(set (reg:CC_Z CC_REGNUM)
2406 (compare:CC_Z
2407 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2408 (match_operand:GPI 1 "register_operand" "r")))]
2409 ""
2410 "cmn\\t%<w>1, %<w>0"
2411 [(set_attr "type" "alus_sreg")]
2412 )
2413
2414 (define_insn "*add_<shift>_<mode>"
2415 [(set (match_operand:GPI 0 "register_operand" "=r")
2416 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2417 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2418 (match_operand:GPI 3 "register_operand" "r")))]
2419 ""
2420 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2421 [(set_attr "type" "alu_shift_imm")]
2422 )
2423
2424 ;; zero_extend version of above
2425 (define_insn "*add_<shift>_si_uxtw"
2426 [(set (match_operand:DI 0 "register_operand" "=r")
2427 (zero_extend:DI
2428 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2429 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2430 (match_operand:SI 3 "register_operand" "r"))))]
2431 ""
2432 "add\\t%w0, %w3, %w1, <shift> %2"
2433 [(set_attr "type" "alu_shift_imm")]
2434 )
2435
2436 (define_insn "*add_mul_imm_<mode>"
2437 [(set (match_operand:GPI 0 "register_operand" "=r")
2438 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2439 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2440 (match_operand:GPI 3 "register_operand" "r")))]
2441 ""
2442 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2443 [(set_attr "type" "alu_shift_imm")]
2444 )
2445
2446 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2447 [(set (match_operand:GPI 0 "register_operand" "=rk")
2448 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2449 (match_operand:GPI 2 "register_operand" "r")))]
2450 ""
2451 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2452 [(set_attr "type" "alu_ext")]
2453 )
2454
2455 ;; zero_extend version of above
2456 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2457 [(set (match_operand:DI 0 "register_operand" "=rk")
2458 (zero_extend:DI
2459 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2460 (match_operand:GPI 2 "register_operand" "r"))))]
2461 ""
2462 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2463 [(set_attr "type" "alu_ext")]
2464 )
2465
2466 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2467 [(set (match_operand:GPI 0 "register_operand" "=rk")
2468 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2469 (match_operand:ALLX 1 "register_operand" "r"))
2470 (match_operand 2 "aarch64_imm3" "Ui3"))
2471 (match_operand:GPI 3 "register_operand" "r")))]
2472 ""
2473 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2474 [(set_attr "type" "alu_ext")]
2475 )
2476
2477 ;; zero_extend version of above
2478 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2479 [(set (match_operand:DI 0 "register_operand" "=rk")
2480 (zero_extend:DI
2481 (plus:SI (ashift:SI (ANY_EXTEND:SI
2482 (match_operand:SHORT 1 "register_operand" "r"))
2483 (match_operand 2 "aarch64_imm3" "Ui3"))
2484 (match_operand:SI 3 "register_operand" "r"))))]
2485 ""
2486 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2487 [(set_attr "type" "alu_ext")]
2488 )
2489
2490 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2491 [(set (match_operand:GPI 0 "register_operand" "=rk")
2492 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2493 (match_operand:ALLX 1 "register_operand" "r"))
2494 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2495 (match_operand:GPI 3 "register_operand" "r")))]
2496 ""
2497 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2498 [(set_attr "type" "alu_ext")]
2499 )
2500
2501 ;; zero_extend version of above
2502 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2503 [(set (match_operand:DI 0 "register_operand" "=rk")
2504 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2505 (match_operand:SHORT 1 "register_operand" "r"))
2506 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2507 (match_operand:SI 3 "register_operand" "r"))))]
2508 ""
2509 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2510 [(set_attr "type" "alu_ext")]
2511 )
2512
2513 (define_insn "*add_<optab><mode>_multp2"
2514 [(set (match_operand:GPI 0 "register_operand" "=rk")
2515 (plus:GPI (ANY_EXTRACT:GPI
2516 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2517 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2518 (match_operand 3 "const_int_operand" "n")
2519 (const_int 0))
2520 (match_operand:GPI 4 "register_operand" "r")))]
2521 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2522 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2523 [(set_attr "type" "alu_ext")]
2524 )
2525
2526 ;; zero_extend version of above
2527 (define_insn "*add_<optab>si_multp2_uxtw"
2528 [(set (match_operand:DI 0 "register_operand" "=rk")
2529 (zero_extend:DI
2530 (plus:SI (ANY_EXTRACT:SI
2531 (mult:SI (match_operand:SI 1 "register_operand" "r")
2532 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2533 (match_operand 3 "const_int_operand" "n")
2534 (const_int 0))
2535 (match_operand:SI 4 "register_operand" "r"))))]
2536 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2537 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2538 [(set_attr "type" "alu_ext")]
2539 )
2540
2541 (define_expand "add<mode>3_carryin"
2542 [(set (match_operand:GPI 0 "register_operand")
2543 (plus:GPI
2544 (plus:GPI
2545 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2546 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2547 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2548 ""
2549 ""
2550 )
2551
2552 ;; Note that add with carry with two zero inputs is matched by cset,
2553 ;; and that add with carry with one zero input is matched by cinc.
2554
2555 (define_insn "*add<mode>3_carryin"
2556 [(set (match_operand:GPI 0 "register_operand" "=r")
2557 (plus:GPI
2558 (plus:GPI
2559 (match_operand:GPI 3 "aarch64_carry_operation" "")
2560 (match_operand:GPI 1 "register_operand" "r"))
2561 (match_operand:GPI 2 "register_operand" "r")))]
2562 ""
2563 "adc\\t%<w>0, %<w>1, %<w>2"
2564 [(set_attr "type" "adc_reg")]
2565 )
2566
2567 ;; zero_extend version of above
2568 (define_insn "*addsi3_carryin_uxtw"
2569 [(set (match_operand:DI 0 "register_operand" "=r")
2570 (zero_extend:DI
2571 (plus:SI
2572 (plus:SI
2573 (match_operand:SI 3 "aarch64_carry_operation" "")
2574 (match_operand:SI 1 "register_operand" "r"))
2575 (match_operand:SI 2 "register_operand" "r"))))]
2576 ""
2577 "adc\\t%w0, %w1, %w2"
2578 [(set_attr "type" "adc_reg")]
2579 )
2580
2581 (define_expand "add<mode>3_carryinC"
2582 [(parallel
2583 [(set (match_dup 3)
2584 (compare:CC_ADC
2585 (plus:<DWI>
2586 (plus:<DWI>
2587 (match_dup 4)
2588 (zero_extend:<DWI>
2589 (match_operand:GPI 1 "register_operand")))
2590 (zero_extend:<DWI>
2591 (match_operand:GPI 2 "register_operand")))
2592 (match_dup 6)))
2593 (set (match_operand:GPI 0 "register_operand")
2594 (plus:GPI
2595 (plus:GPI (match_dup 5) (match_dup 1))
2596 (match_dup 2)))])]
2597 ""
2598 {
2599 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2600 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2601 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2602 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2603 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2604 << GET_MODE_BITSIZE (<MODE>mode),
2605 TImode);
2606 })
2607
2608 (define_insn "*add<mode>3_carryinC_zero"
2609 [(set (reg:CC_ADC CC_REGNUM)
2610 (compare:CC_ADC
2611 (plus:<DWI>
2612 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2613 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2614 (match_operand 4 "const_scalar_int_operand" "")))
2615 (set (match_operand:GPI 0 "register_operand" "=r")
2616 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2617 (match_dup 1)))]
2618 "rtx_mode_t (operands[4], <DWI>mode)
2619 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2620 "adcs\\t%<w>0, %<w>1, <w>zr"
2621 [(set_attr "type" "adc_reg")]
2622 )
2623
2624 (define_insn "*add<mode>3_carryinC"
2625 [(set (reg:CC_ADC CC_REGNUM)
2626 (compare:CC_ADC
2627 (plus:<DWI>
2628 (plus:<DWI>
2629 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2630 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2631 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2632 (match_operand 5 "const_scalar_int_operand" "")))
2633 (set (match_operand:GPI 0 "register_operand" "=r")
2634 (plus:GPI
2635 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2636 (match_dup 1))
2637 (match_dup 2)))]
2638 "rtx_mode_t (operands[5], <DWI>mode)
2639 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2640 "adcs\\t%<w>0, %<w>1, %<w>2"
2641 [(set_attr "type" "adc_reg")]
2642 )
2643
2644 (define_expand "add<mode>3_carryinV"
2645 [(parallel
2646 [(set (reg:CC_V CC_REGNUM)
2647 (compare:CC_V
2648 (plus:<DWI>
2649 (plus:<DWI>
2650 (match_dup 3)
2651 (sign_extend:<DWI>
2652 (match_operand:GPI 1 "register_operand")))
2653 (sign_extend:<DWI>
2654 (match_operand:GPI 2 "register_operand")))
2655 (sign_extend:<DWI>
2656 (plus:GPI
2657 (plus:GPI (match_dup 4) (match_dup 1))
2658 (match_dup 2)))))
2659 (set (match_operand:GPI 0 "register_operand")
2660 (plus:GPI
2661 (plus:GPI (match_dup 4) (match_dup 1))
2662 (match_dup 2)))])]
2663 ""
2664 {
2665 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2666 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2667 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2668 })
2669
2670 (define_insn "*add<mode>3_carryinV_zero"
2671 [(set (reg:CC_V CC_REGNUM)
2672 (compare:CC_V
2673 (plus:<DWI>
2674 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2675 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2676 (sign_extend:<DWI>
2677 (plus:GPI
2678 (match_operand:GPI 3 "aarch64_carry_operation" "")
2679 (match_dup 1)))))
2680 (set (match_operand:GPI 0 "register_operand" "=r")
2681 (plus:GPI (match_dup 3) (match_dup 1)))]
2682 ""
2683 "adcs\\t%<w>0, %<w>1, <w>zr"
2684 [(set_attr "type" "adc_reg")]
2685 )
2686
2687 (define_insn "*add<mode>3_carryinV"
2688 [(set (reg:CC_V CC_REGNUM)
2689 (compare:CC_V
2690 (plus:<DWI>
2691 (plus:<DWI>
2692 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2693 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2694 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2695 (sign_extend:<DWI>
2696 (plus:GPI
2697 (plus:GPI
2698 (match_operand:GPI 4 "aarch64_carry_operation" "")
2699 (match_dup 1))
2700 (match_dup 2)))))
2701 (set (match_operand:GPI 0 "register_operand" "=r")
2702 (plus:GPI
2703 (plus:GPI (match_dup 4) (match_dup 1))
2704 (match_dup 2)))]
2705 ""
2706 "adcs\\t%<w>0, %<w>1, %<w>2"
2707 [(set_attr "type" "adc_reg")]
2708 )
2709
2710 (define_insn "*add_uxt<mode>_shift2"
2711 [(set (match_operand:GPI 0 "register_operand" "=rk")
2712 (plus:GPI (and:GPI
2713 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2714 (match_operand 2 "aarch64_imm3" "Ui3"))
2715 (match_operand 3 "const_int_operand" "n"))
2716 (match_operand:GPI 4 "register_operand" "r")))]
2717 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2718 "*
2719 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2720 INTVAL (operands[3])));
2721 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2722 [(set_attr "type" "alu_ext")]
2723 )
2724
2725 ;; zero_extend version of above
2726 (define_insn "*add_uxtsi_shift2_uxtw"
2727 [(set (match_operand:DI 0 "register_operand" "=rk")
2728 (zero_extend:DI
2729 (plus:SI (and:SI
2730 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2731 (match_operand 2 "aarch64_imm3" "Ui3"))
2732 (match_operand 3 "const_int_operand" "n"))
2733 (match_operand:SI 4 "register_operand" "r"))))]
2734 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2735 "*
2736 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2737 INTVAL (operands[3])));
2738 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2739 [(set_attr "type" "alu_ext")]
2740 )
2741
2742 (define_insn "*add_uxt<mode>_multp2"
2743 [(set (match_operand:GPI 0 "register_operand" "=rk")
2744 (plus:GPI (and:GPI
2745 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2746 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2747 (match_operand 3 "const_int_operand" "n"))
2748 (match_operand:GPI 4 "register_operand" "r")))]
2749 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2750 "*
2751 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2752 INTVAL (operands[3])));
2753 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2754 [(set_attr "type" "alu_ext")]
2755 )
2756
2757 ;; zero_extend version of above
2758 (define_insn "*add_uxtsi_multp2_uxtw"
2759 [(set (match_operand:DI 0 "register_operand" "=rk")
2760 (zero_extend:DI
2761 (plus:SI (and:SI
2762 (mult:SI (match_operand:SI 1 "register_operand" "r")
2763 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2764 (match_operand 3 "const_int_operand" "n"))
2765 (match_operand:SI 4 "register_operand" "r"))))]
2766 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2767 "*
2768 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2769 INTVAL (operands[3])));
2770 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2771 [(set_attr "type" "alu_ext")]
2772 )
2773
2774 (define_insn "subsi3"
2775 [(set (match_operand:SI 0 "register_operand" "=rk")
2776 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2777 (match_operand:SI 2 "register_operand" "r")))]
2778 ""
2779 "sub\\t%w0, %w1, %w2"
2780 [(set_attr "type" "alu_sreg")]
2781 )
2782
2783 ;; zero_extend version of above
2784 (define_insn "*subsi3_uxtw"
2785 [(set (match_operand:DI 0 "register_operand" "=rk")
2786 (zero_extend:DI
2787 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2788 (match_operand:SI 2 "register_operand" "r"))))]
2789 ""
2790 "sub\\t%w0, %w1, %w2"
2791 [(set_attr "type" "alu_sreg")]
2792 )
2793
2794 (define_insn "subdi3"
2795 [(set (match_operand:DI 0 "register_operand" "=rk,w")
2796 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2797 (match_operand:DI 2 "register_operand" "r,w")))]
2798 ""
2799 "@
2800 sub\\t%x0, %x1, %x2
2801 sub\\t%d0, %d1, %d2"
2802 [(set_attr "type" "alu_sreg, neon_sub")
2803 (set_attr "arch" "*,simd")]
2804 )
2805
2806 (define_expand "subv<GPI:mode>4"
2807 [(match_operand:GPI 0 "register_operand")
2808 (match_operand:GPI 1 "register_operand")
2809 (match_operand:GPI 2 "aarch64_plus_operand")
2810 (label_ref (match_operand 3 "" ""))]
2811 ""
2812 {
2813 if (CONST_INT_P (operands[2]))
2814 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2815 else
2816 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2817 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2818
2819 DONE;
2820 })
2821
2822 (define_insn "subv<GPI:mode>_insn"
2823 [(set (reg:CC_V CC_REGNUM)
2824 (compare:CC_V
2825 (sign_extend:<DWI>
2826 (minus:GPI
2827 (match_operand:GPI 1 "register_operand" "rk")
2828 (match_operand:GPI 2 "register_operand" "r")))
2829 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2830 (sign_extend:<DWI> (match_dup 2)))))
2831 (set (match_operand:GPI 0 "register_operand" "=r")
2832 (minus:GPI (match_dup 1) (match_dup 2)))]
2833 ""
2834 "subs\\t%<w>0, %<w>1, %<w>2"
2835 [(set_attr "type" "alus_sreg")]
2836 )
2837
2838 (define_insn "subv<GPI:mode>_imm"
2839 [(set (reg:CC_V CC_REGNUM)
2840 (compare:CC_V
2841 (sign_extend:<DWI>
2842 (minus:GPI
2843 (match_operand:GPI 1 "register_operand" "rk,rk")
2844 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2845 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2846 (match_dup 2))))
2847 (set (match_operand:GPI 0 "register_operand" "=r,r")
2848 (minus:GPI (match_dup 1) (match_dup 2)))]
2849 ""
2850 "@
2851 subs\\t%<w>0, %<w>1, %2
2852 adds\\t%<w>0, %<w>1, #%n2"
2853 [(set_attr "type" "alus_sreg")]
2854 )
2855
2856 (define_expand "negv<GPI:mode>3"
2857 [(match_operand:GPI 0 "register_operand")
2858 (match_operand:GPI 1 "register_operand")
2859 (label_ref (match_operand 2 "" ""))]
2860 ""
2861 {
2862 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2863 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2864
2865 DONE;
2866 }
2867 )
2868
2869 (define_insn "negv<GPI:mode>_insn"
2870 [(set (reg:CC_V CC_REGNUM)
2871 (compare:CC_V
2872 (sign_extend:<DWI>
2873 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2874 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2875 (set (match_operand:GPI 0 "register_operand" "=r")
2876 (neg:GPI (match_dup 1)))]
2877 ""
2878 "negs\\t%<w>0, %<w>1"
2879 [(set_attr "type" "alus_sreg")]
2880 )
2881
2882 (define_insn "negv<GPI:mode>_cmp_only"
2883 [(set (reg:CC_V CC_REGNUM)
2884 (compare:CC_V
2885 (sign_extend:<DWI>
2886 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2887 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2888 ""
2889 "negs\\t%<w>zr, %<w>0"
2890 [(set_attr "type" "alus_sreg")]
2891 )
2892
2893 (define_insn "*cmpv<GPI:mode>_insn"
2894 [(set (reg:CC_V CC_REGNUM)
2895 (compare:CC_V
2896 (sign_extend:<DWI>
2897 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2898 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2899 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2900 (sign_extend:<DWI> (match_dup 1)))))]
2901 ""
2902 "@
2903 cmp\\t%<w>0, %<w>1
2904 cmp\\t%<w>0, %1
2905 cmp\\t%<w>0, #%n1"
2906 [(set_attr "type" "alus_sreg")]
2907 )
2908
2909 (define_expand "usubv<mode>4"
2910 [(match_operand:GPI 0 "register_operand")
2911 (match_operand:GPI 1 "aarch64_reg_or_zero")
2912 (match_operand:GPI 2 "aarch64_reg_or_zero")
2913 (label_ref (match_operand 3 "" ""))]
2914 ""
2915 {
2916 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2917 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2918
2919 DONE;
2920 })
2921
2922 (define_expand "subti3"
2923 [(set (match_operand:TI 0 "register_operand")
2924 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2925 (match_operand:TI 2 "register_operand")))]
2926 ""
2927 {
2928 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2929
2930 aarch64_subvti_scratch_regs (operands[1], operands[2],
2931 &low_dest, &op1_low, &op2_low,
2932 &high_dest, &op1_high, &op2_high);
2933
2934 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2935 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2936
2937 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2938 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2939 DONE;
2940 })
2941
2942 (define_expand "subvti4"
2943 [(match_operand:TI 0 "register_operand")
2944 (match_operand:TI 1 "register_operand")
2945 (match_operand:TI 2 "aarch64_reg_or_imm")
2946 (label_ref (match_operand 3 "" ""))]
2947 ""
2948 {
2949 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2950
2951 aarch64_subvti_scratch_regs (operands[1], operands[2],
2952 &low_dest, &op1_low, &op2_low,
2953 &high_dest, &op1_high, &op2_high);
2954 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2955 high_dest, op1_high, op2_high, false);
2956
2957 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2958 DONE;
2959 })
2960
2961 (define_expand "usubvti4"
2962 [(match_operand:TI 0 "register_operand")
2963 (match_operand:TI 1 "register_operand")
2964 (match_operand:TI 2 "aarch64_reg_or_imm")
2965 (label_ref (match_operand 3 "" ""))]
2966 ""
2967 {
2968 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2969
2970 aarch64_subvti_scratch_regs (operands[1], operands[2],
2971 &low_dest, &op1_low, &op2_low,
2972 &high_dest, &op1_high, &op2_high);
2973 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2974 high_dest, op1_high, op2_high, true);
2975
2976 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2977 DONE;
2978 })
2979
2980 (define_expand "negvti3"
2981 [(match_operand:TI 0 "register_operand")
2982 (match_operand:TI 1 "register_operand")
2983 (label_ref (match_operand 2 "" ""))]
2984 ""
2985 {
2986 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2987 gen_lowpart (DImode, operands[1])));
2988 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2989 gen_highpart (DImode, operands[1])));
2990 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2991
2992 DONE;
2993 }
2994 )
2995
2996 (define_insn "negdi_carryout"
2997 [(set (reg:CC CC_REGNUM)
2998 (compare:CC
2999 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
3000 (set (match_operand:DI 0 "register_operand" "=r")
3001 (neg:DI (match_dup 1)))]
3002 ""
3003 "negs\\t%0, %1"
3004 [(set_attr "type" "alus_sreg")]
3005 )
3006
3007 (define_insn "negvdi_carryinV"
3008 [(set (reg:CC_V CC_REGNUM)
3009 (compare:CC_V
3010 (neg:TI (plus:TI
3011 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
3012 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
3013 (sign_extend:TI
3014 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3015 (match_dup 1))))))
3016 (set (match_operand:DI 0 "register_operand" "=r")
3017 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3018 (match_dup 1))))]
3019 ""
3020 "ngcs\\t%0, %1"
3021 [(set_attr "type" "alus_sreg")]
3022 )
3023
3024 (define_insn "*sub<mode>3_compare0"
3025 [(set (reg:CC_NZ CC_REGNUM)
3026 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3027 (match_operand:GPI 2 "register_operand" "r"))
3028 (const_int 0)))
3029 (set (match_operand:GPI 0 "register_operand" "=r")
3030 (minus:GPI (match_dup 1) (match_dup 2)))]
3031 ""
3032 "subs\\t%<w>0, %<w>1, %<w>2"
3033 [(set_attr "type" "alus_sreg")]
3034 )
3035
3036 ;; zero_extend version of above
3037 (define_insn "*subsi3_compare0_uxtw"
3038 [(set (reg:CC_NZ CC_REGNUM)
3039 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3040 (match_operand:SI 2 "register_operand" "r"))
3041 (const_int 0)))
3042 (set (match_operand:DI 0 "register_operand" "=r")
3043 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3044 ""
3045 "subs\\t%w0, %w1, %w2"
3046 [(set_attr "type" "alus_sreg")]
3047 )
3048
3049 (define_insn "sub<mode>3_compare1_imm"
3050 [(set (reg:CC CC_REGNUM)
3051 (compare:CC
3052 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
3053 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3054 (set (match_operand:GPI 0 "register_operand" "=r,r")
3055 (plus:GPI
3056 (match_dup 1)
3057 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3058 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3059 "@
3060 subs\\t%<w>0, %<w>1, %2
3061 adds\\t%<w>0, %<w>1, #%n2"
3062 [(set_attr "type" "alus_imm")]
3063 )
3064
3065 (define_insn "sub<mode>3_compare1"
3066 [(set (reg:CC CC_REGNUM)
3067 (compare:CC
3068 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3069 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3070 (set (match_operand:GPI 0 "register_operand" "=r")
3071 (minus:GPI (match_dup 1) (match_dup 2)))]
3072 ""
3073 "subs\\t%<w>0, %<w>1, %<w>2"
3074 [(set_attr "type" "alus_sreg")]
3075 )
3076
3077 (define_peephole2
3078 [(set (match_operand:GPI 0 "aarch64_general_reg")
3079 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3080 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3081 (set (reg:CC CC_REGNUM)
3082 (compare:CC
3083 (match_dup 1)
3084 (match_dup 2)))]
3085 "!reg_overlap_mentioned_p (operands[0], operands[1])
3086 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3087 [(const_int 0)]
3088 {
3089 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3090 operands[2]));
3091 DONE;
3092 }
3093 )
3094
3095 ;; Same as the above peephole but with the compare and minus in
3096 ;; swapped order. The restriction on overlap between operand 0
3097 ;; and operands 1 and 2 doesn't apply here.
3098 (define_peephole2
3099 [(set (reg:CC CC_REGNUM)
3100 (compare:CC
3101 (match_operand:GPI 1 "aarch64_reg_or_zero")
3102 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3103 (set (match_operand:GPI 0 "aarch64_general_reg")
3104 (minus:GPI (match_dup 1)
3105 (match_dup 2)))]
3106 ""
3107 [(const_int 0)]
3108 {
3109 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3110 operands[2]));
3111 DONE;
3112 }
3113 )
3114
3115 (define_peephole2
3116 [(set (match_operand:GPI 0 "aarch64_general_reg")
3117 (plus:GPI (match_operand:GPI 1 "register_operand")
3118 (match_operand:GPI 2 "aarch64_plus_immediate")))
3119 (set (reg:CC CC_REGNUM)
3120 (compare:CC
3121 (match_dup 1)
3122 (match_operand:GPI 3 "const_int_operand")))]
3123 "!reg_overlap_mentioned_p (operands[0], operands[1])
3124 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3125 [(const_int 0)]
3126 {
3127 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3128 operands[3], operands[2]));
3129 DONE;
3130 }
3131 )
3132
3133 ;; Same as the above peephole but with the compare and minus in
3134 ;; swapped order. The restriction on overlap between operand 0
3135 ;; and operands 1 doesn't apply here.
3136 (define_peephole2
3137 [(set (reg:CC CC_REGNUM)
3138 (compare:CC
3139 (match_operand:GPI 1 "register_operand")
3140 (match_operand:GPI 3 "const_int_operand")))
3141 (set (match_operand:GPI 0 "aarch64_general_reg")
3142 (plus:GPI (match_dup 1)
3143 (match_operand:GPI 2 "aarch64_plus_immediate")))]
3144 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3145 [(const_int 0)]
3146 {
3147 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3148 operands[3], operands[2]));
3149 DONE;
3150 }
3151 )
3152
3153 (define_insn "*sub_<shift>_<mode>"
3154 [(set (match_operand:GPI 0 "register_operand" "=r")
3155 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3156 (ASHIFT:GPI
3157 (match_operand:GPI 1 "register_operand" "r")
3158 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3159 ""
3160 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3161 [(set_attr "type" "alu_shift_imm")]
3162 )
3163
3164 ;; zero_extend version of above
3165 (define_insn "*sub_<shift>_si_uxtw"
3166 [(set (match_operand:DI 0 "register_operand" "=r")
3167 (zero_extend:DI
3168 (minus:SI (match_operand:SI 3 "register_operand" "r")
3169 (ASHIFT:SI
3170 (match_operand:SI 1 "register_operand" "r")
3171 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3172 ""
3173 "sub\\t%w0, %w3, %w1, <shift> %2"
3174 [(set_attr "type" "alu_shift_imm")]
3175 )
3176
3177 (define_insn "*sub_mul_imm_<mode>"
3178 [(set (match_operand:GPI 0 "register_operand" "=r")
3179 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3180 (mult:GPI
3181 (match_operand:GPI 1 "register_operand" "r")
3182 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3183 ""
3184 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3185 [(set_attr "type" "alu_shift_imm")]
3186 )
3187
3188 ;; zero_extend version of above
3189 (define_insn "*sub_mul_imm_si_uxtw"
3190 [(set (match_operand:DI 0 "register_operand" "=r")
3191 (zero_extend:DI
3192 (minus:SI (match_operand:SI 3 "register_operand" "r")
3193 (mult:SI
3194 (match_operand:SI 1 "register_operand" "r")
3195 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3196 ""
3197 "sub\\t%w0, %w3, %w1, lsl %p2"
3198 [(set_attr "type" "alu_shift_imm")]
3199 )
3200
3201 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3202 [(set (match_operand:GPI 0 "register_operand" "=rk")
3203 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3204 (ANY_EXTEND:GPI
3205 (match_operand:ALLX 2 "register_operand" "r"))))]
3206 ""
3207 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3208 [(set_attr "type" "alu_ext")]
3209 )
3210
3211 ;; zero_extend version of above
3212 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3213 [(set (match_operand:DI 0 "register_operand" "=rk")
3214 (zero_extend:DI
3215 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3216 (ANY_EXTEND:SI
3217 (match_operand:SHORT 2 "register_operand" "r")))))]
3218 ""
3219 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3220 [(set_attr "type" "alu_ext")]
3221 )
3222
3223 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3224 [(set (match_operand:GPI 0 "register_operand" "=rk")
3225 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3226 (ashift:GPI (ANY_EXTEND:GPI
3227 (match_operand:ALLX 2 "register_operand" "r"))
3228 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3229 ""
3230 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3231 [(set_attr "type" "alu_ext")]
3232 )
3233
3234 ;; zero_extend version of above
3235 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3236 [(set (match_operand:DI 0 "register_operand" "=rk")
3237 (zero_extend:DI
3238 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3239 (ashift:SI (ANY_EXTEND:SI
3240 (match_operand:SHORT 2 "register_operand" "r"))
3241 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3242 ""
3243 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3244 [(set_attr "type" "alu_ext")]
3245 )
3246
3247 (define_insn "*sub_<optab><mode>_multp2"
3248 [(set (match_operand:GPI 0 "register_operand" "=rk")
3249 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3250 (ANY_EXTRACT:GPI
3251 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3252 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3253 (match_operand 3 "const_int_operand" "n")
3254 (const_int 0))))]
3255 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3256 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3257 [(set_attr "type" "alu_ext")]
3258 )
3259
3260 ;; zero_extend version of above
3261 (define_insn "*sub_<optab>si_multp2_uxtw"
3262 [(set (match_operand:DI 0 "register_operand" "=rk")
3263 (zero_extend:DI
3264 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3265 (ANY_EXTRACT:SI
3266 (mult:SI (match_operand:SI 1 "register_operand" "r")
3267 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3268 (match_operand 3 "const_int_operand" "n")
3269 (const_int 0)))))]
3270 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3271 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3272 [(set_attr "type" "alu_ext")]
3273 )
3274
3275 ;; The hardware description is op1 + ~op2 + C.
3276 ;; = op1 + (-op2 + 1) + (1 - !C)
3277 ;; = op1 - op2 - 1 + 1 - !C
3278 ;; = op1 - op2 - !C.
3279 ;; We describe the latter.
3280
3281 (define_insn "*sub<mode>3_carryin0"
3282 [(set (match_operand:GPI 0 "register_operand" "=r")
3283 (minus:GPI
3284 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3285 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3286 ""
3287 "sbc\\t%<w>0, %<w>1, <w>zr"
3288 [(set_attr "type" "adc_reg")]
3289 )
3290
3291 ;; zero_extend version of the above
3292 (define_insn "*subsi3_carryin_uxtw"
3293 [(set (match_operand:DI 0 "register_operand" "=r")
3294 (zero_extend:DI
3295 (minus:SI
3296 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3297 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3298 ""
3299 "sbc\\t%w0, %w1, wzr"
3300 [(set_attr "type" "adc_reg")]
3301 )
3302
3303 (define_expand "sub<mode>3_carryin"
3304 [(set (match_operand:GPI 0 "register_operand")
3305 (minus:GPI
3306 (minus:GPI
3307 (match_operand:GPI 1 "aarch64_reg_or_zero")
3308 (match_operand:GPI 2 "register_operand"))
3309 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3310 ""
3311 ""
3312 )
3313
3314 (define_insn "*sub<mode>3_carryin"
3315 [(set (match_operand:GPI 0 "register_operand" "=r")
3316 (minus:GPI
3317 (minus:GPI
3318 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3319 (match_operand:GPI 2 "register_operand" "r"))
3320 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3321
3322 ""
3323 "sbc\\t%<w>0, %<w>1, %<w>2"
3324 [(set_attr "type" "adc_reg")]
3325 )
3326
3327 ;; zero_extend version of the above
3328 (define_insn "*subsi3_carryin_uxtw"
3329 [(set (match_operand:DI 0 "register_operand" "=r")
3330 (zero_extend:DI
3331 (minus:SI
3332 (minus:SI
3333 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3334 (match_operand:SI 2 "register_operand" "r"))
3335 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3336
3337 ""
3338 "sbc\\t%w0, %w1, %w2"
3339 [(set_attr "type" "adc_reg")]
3340 )
3341
3342 (define_insn "*sub<mode>3_carryin_alt"
3343 [(set (match_operand:GPI 0 "register_operand" "=r")
3344 (minus:GPI
3345 (minus:GPI
3346 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3347 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3348 (match_operand:GPI 2 "register_operand" "r")))]
3349 ""
3350 "sbc\\t%<w>0, %<w>1, %<w>2"
3351 [(set_attr "type" "adc_reg")]
3352 )
3353
3354 ;; zero_extend version of the above
3355 (define_insn "*subsi3_carryin_alt_uxtw"
3356 [(set (match_operand:DI 0 "register_operand" "=r")
3357 (zero_extend:DI
3358 (minus:SI
3359 (minus:SI
3360 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3361 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3362 (match_operand:SI 2 "register_operand" "r"))))]
3363 ""
3364 "sbc\\t%w0, %w1, %w2"
3365 [(set_attr "type" "adc_reg")]
3366 )
3367
3368 (define_expand "usub<GPI:mode>3_carryinC"
3369 [(parallel
3370 [(set (reg:CC CC_REGNUM)
3371 (compare:CC
3372 (zero_extend:<DWI>
3373 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3374 (plus:<DWI>
3375 (zero_extend:<DWI>
3376 (match_operand:GPI 2 "register_operand"))
3377 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3378 (set (match_operand:GPI 0 "register_operand")
3379 (minus:GPI
3380 (minus:GPI (match_dup 1) (match_dup 2))
3381 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3382 ""
3383 )
3384
3385 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3386 [(set (reg:CC CC_REGNUM)
3387 (compare:CC
3388 (const_int 0)
3389 (plus:<DWI>
3390 (zero_extend:<DWI>
3391 (match_operand:GPI 1 "register_operand" "r"))
3392 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3393 (set (match_operand:GPI 0 "register_operand" "=r")
3394 (minus:GPI
3395 (neg:GPI (match_dup 1))
3396 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3397 ""
3398 "sbcs\\t%<w>0, <w>zr, %<w>1"
3399 [(set_attr "type" "adc_reg")]
3400 )
3401
3402 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3403 [(set (reg:CC CC_REGNUM)
3404 (compare:CC
3405 (zero_extend:<DWI>
3406 (match_operand:GPI 1 "register_operand" "r"))
3407 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3408 (set (match_operand:GPI 0 "register_operand" "=r")
3409 (minus:GPI
3410 (match_dup 1)
3411 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3412 ""
3413 "sbcs\\t%<w>0, %<w>1, <w>zr"
3414 [(set_attr "type" "adc_reg")]
3415 )
3416
3417 (define_insn "*usub<GPI:mode>3_carryinC"
3418 [(set (reg:CC CC_REGNUM)
3419 (compare:CC
3420 (zero_extend:<DWI>
3421 (match_operand:GPI 1 "register_operand" "r"))
3422 (plus:<DWI>
3423 (zero_extend:<DWI>
3424 (match_operand:GPI 2 "register_operand" "r"))
3425 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3426 (set (match_operand:GPI 0 "register_operand" "=r")
3427 (minus:GPI
3428 (minus:GPI (match_dup 1) (match_dup 2))
3429 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3430 ""
3431 "sbcs\\t%<w>0, %<w>1, %<w>2"
3432 [(set_attr "type" "adc_reg")]
3433 )
3434
3435 (define_expand "sub<GPI:mode>3_carryinV"
3436 [(parallel
3437 [(set (reg:CC_V CC_REGNUM)
3438 (compare:CC_V
3439 (minus:<DWI>
3440 (sign_extend:<DWI>
3441 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3442 (plus:<DWI>
3443 (sign_extend:<DWI>
3444 (match_operand:GPI 2 "register_operand"))
3445 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3446 (sign_extend:<DWI>
3447 (minus:GPI (match_dup 1)
3448 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3449 (match_dup 2))))))
3450 (set (match_operand:GPI 0 "register_operand")
3451 (minus:GPI
3452 (minus:GPI (match_dup 1) (match_dup 2))
3453 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3454 ""
3455 )
3456
3457 (define_insn "*sub<mode>3_carryinV_z2"
3458 [(set (reg:CC_V CC_REGNUM)
3459 (compare:CC_V
3460 (minus:<DWI>
3461 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3462 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3463 (sign_extend:<DWI>
3464 (minus:GPI (match_dup 1)
3465 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3466 (set (match_operand:GPI 0 "register_operand" "=r")
3467 (minus:GPI
3468 (match_dup 1) (match_dup 3)))]
3469 ""
3470 "sbcs\\t%<w>0, %<w>1, <w>zr"
3471 [(set_attr "type" "adc_reg")]
3472 )
3473
3474 (define_insn "*sub<mode>3_carryinV"
3475 [(set (reg:CC_V CC_REGNUM)
3476 (compare:CC_V
3477 (minus:<DWI>
3478 (sign_extend:<DWI>
3479 (match_operand:GPI 1 "register_operand" "r"))
3480 (plus:<DWI>
3481 (sign_extend:<DWI>
3482 (match_operand:GPI 2 "register_operand" "r"))
3483 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3484 (sign_extend:<DWI>
3485 (minus:GPI
3486 (match_dup 1)
3487 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3488 (match_dup 2))))))
3489 (set (match_operand:GPI 0 "register_operand" "=r")
3490 (minus:GPI
3491 (minus:GPI (match_dup 1) (match_dup 2))
3492 (match_dup 4)))]
3493 ""
3494 "sbcs\\t%<w>0, %<w>1, %<w>2"
3495 [(set_attr "type" "adc_reg")]
3496 )
3497
3498 (define_insn "*sub_uxt<mode>_shift2"
3499 [(set (match_operand:GPI 0 "register_operand" "=rk")
3500 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3501 (and:GPI
3502 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3503 (match_operand 2 "aarch64_imm3" "Ui3"))
3504 (match_operand 3 "const_int_operand" "n"))))]
3505 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3506 "*
3507 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3508 INTVAL (operands[3])));
3509 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3510 [(set_attr "type" "alu_ext")]
3511 )
3512
3513 ;; zero_extend version of above
3514 (define_insn "*sub_uxtsi_shift2_uxtw"
3515 [(set (match_operand:DI 0 "register_operand" "=rk")
3516 (zero_extend:DI
3517 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3518 (and:SI
3519 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3520 (match_operand 2 "aarch64_imm3" "Ui3"))
3521 (match_operand 3 "const_int_operand" "n")))))]
3522 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3523 "*
3524 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3525 INTVAL (operands[3])));
3526 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3527 [(set_attr "type" "alu_ext")]
3528 )
3529
3530 (define_insn "*sub_uxt<mode>_multp2"
3531 [(set (match_operand:GPI 0 "register_operand" "=rk")
3532 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3533 (and:GPI
3534 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3535 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3536 (match_operand 3 "const_int_operand" "n"))))]
3537 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3538 "*
3539 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3540 INTVAL (operands[3])));
3541 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3542 [(set_attr "type" "alu_ext")]
3543 )
3544
3545 ;; zero_extend version of above
3546 (define_insn "*sub_uxtsi_multp2_uxtw"
3547 [(set (match_operand:DI 0 "register_operand" "=rk")
3548 (zero_extend:DI
3549 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3550 (and:SI
3551 (mult:SI (match_operand:SI 1 "register_operand" "r")
3552 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3553 (match_operand 3 "const_int_operand" "n")))))]
3554 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3555 "*
3556 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3557 INTVAL (operands[3])));
3558 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3559 [(set_attr "type" "alu_ext")]
3560 )
3561
3562 (define_expand "abs<mode>2"
3563 [(match_operand:GPI 0 "register_operand")
3564 (match_operand:GPI 1 "register_operand")]
3565 ""
3566 {
3567 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3568 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3569 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3570 DONE;
3571 }
3572 )
3573
3574 (define_insn "neg<mode>2"
3575 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3576 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3577 ""
3578 "@
3579 neg\\t%<w>0, %<w>1
3580 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3581 [(set_attr "type" "alu_sreg, neon_neg<q>")
3582 (set_attr "arch" "*,simd")]
3583 )
3584
3585 ;; zero_extend version of above
3586 (define_insn "*negsi2_uxtw"
3587 [(set (match_operand:DI 0 "register_operand" "=r")
3588 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3589 ""
3590 "neg\\t%w0, %w1"
3591 [(set_attr "type" "alu_sreg")]
3592 )
3593
3594 (define_insn "*ngc<mode>"
3595 [(set (match_operand:GPI 0 "register_operand" "=r")
3596 (minus:GPI
3597 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3598 (match_operand:GPI 1 "register_operand" "r")))]
3599 ""
3600 "ngc\\t%<w>0, %<w>1"
3601 [(set_attr "type" "adc_reg")]
3602 )
3603
3604 (define_insn "*ngcsi_uxtw"
3605 [(set (match_operand:DI 0 "register_operand" "=r")
3606 (zero_extend:DI
3607 (minus:SI
3608 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3609 (match_operand:SI 1 "register_operand" "r"))))]
3610 ""
3611 "ngc\\t%w0, %w1"
3612 [(set_attr "type" "adc_reg")]
3613 )
3614
3615 (define_insn "neg<mode>2_compare0"
3616 [(set (reg:CC_NZ CC_REGNUM)
3617 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3618 (const_int 0)))
3619 (set (match_operand:GPI 0 "register_operand" "=r")
3620 (neg:GPI (match_dup 1)))]
3621 ""
3622 "negs\\t%<w>0, %<w>1"
3623 [(set_attr "type" "alus_sreg")]
3624 )
3625
3626 ;; zero_extend version of above
3627 (define_insn "*negsi2_compare0_uxtw"
3628 [(set (reg:CC_NZ CC_REGNUM)
3629 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3630 (const_int 0)))
3631 (set (match_operand:DI 0 "register_operand" "=r")
3632 (zero_extend:DI (neg:SI (match_dup 1))))]
3633 ""
3634 "negs\\t%w0, %w1"
3635 [(set_attr "type" "alus_sreg")]
3636 )
3637
3638 (define_insn "*neg_<shift><mode>3_compare0"
3639 [(set (reg:CC_NZ CC_REGNUM)
3640 (compare:CC_NZ
3641 (neg:GPI (ASHIFT:GPI
3642 (match_operand:GPI 1 "register_operand" "r")
3643 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3644 (const_int 0)))
3645 (set (match_operand:GPI 0 "register_operand" "=r")
3646 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3647 ""
3648 "negs\\t%<w>0, %<w>1, <shift> %2"
3649 [(set_attr "type" "alus_shift_imm")]
3650 )
3651
3652 (define_insn "*neg_<shift>_<mode>2"
3653 [(set (match_operand:GPI 0 "register_operand" "=r")
3654 (neg:GPI (ASHIFT:GPI
3655 (match_operand:GPI 1 "register_operand" "r")
3656 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3657 ""
3658 "neg\\t%<w>0, %<w>1, <shift> %2"
3659 [(set_attr "type" "alu_shift_imm")]
3660 )
3661
3662 ;; zero_extend version of above
3663 (define_insn "*neg_<shift>_si2_uxtw"
3664 [(set (match_operand:DI 0 "register_operand" "=r")
3665 (zero_extend:DI
3666 (neg:SI (ASHIFT:SI
3667 (match_operand:SI 1 "register_operand" "r")
3668 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3669 ""
3670 "neg\\t%w0, %w1, <shift> %2"
3671 [(set_attr "type" "alu_shift_imm")]
3672 )
3673
3674 (define_insn "*neg_mul_imm_<mode>2"
3675 [(set (match_operand:GPI 0 "register_operand" "=r")
3676 (neg:GPI (mult:GPI
3677 (match_operand:GPI 1 "register_operand" "r")
3678 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3679 ""
3680 "neg\\t%<w>0, %<w>1, lsl %p2"
3681 [(set_attr "type" "alu_shift_imm")]
3682 )
3683
3684 ;; zero_extend version of above
3685 (define_insn "*neg_mul_imm_si2_uxtw"
3686 [(set (match_operand:DI 0 "register_operand" "=r")
3687 (zero_extend:DI
3688 (neg:SI (mult:SI
3689 (match_operand:SI 1 "register_operand" "r")
3690 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3691 ""
3692 "neg\\t%w0, %w1, lsl %p2"
3693 [(set_attr "type" "alu_shift_imm")]
3694 )
3695
3696 (define_insn "mul<mode>3"
3697 [(set (match_operand:GPI 0 "register_operand" "=r")
3698 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3699 (match_operand:GPI 2 "register_operand" "r")))]
3700 ""
3701 "mul\\t%<w>0, %<w>1, %<w>2"
3702 [(set_attr "type" "mul")]
3703 )
3704
3705 ;; zero_extend version of above
3706 (define_insn "*mulsi3_uxtw"
3707 [(set (match_operand:DI 0 "register_operand" "=r")
3708 (zero_extend:DI
3709 (mult:SI (match_operand:SI 1 "register_operand" "r")
3710 (match_operand:SI 2 "register_operand" "r"))))]
3711 ""
3712 "mul\\t%w0, %w1, %w2"
3713 [(set_attr "type" "mul")]
3714 )
3715
3716 (define_insn "madd<mode>"
3717 [(set (match_operand:GPI 0 "register_operand" "=r")
3718 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3719 (match_operand:GPI 2 "register_operand" "r"))
3720 (match_operand:GPI 3 "register_operand" "r")))]
3721 ""
3722 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3723 [(set_attr "type" "mla")]
3724 )
3725
3726 ;; zero_extend version of above
3727 (define_insn "*maddsi_uxtw"
3728 [(set (match_operand:DI 0 "register_operand" "=r")
3729 (zero_extend:DI
3730 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3731 (match_operand:SI 2 "register_operand" "r"))
3732 (match_operand:SI 3 "register_operand" "r"))))]
3733 ""
3734 "madd\\t%w0, %w1, %w2, %w3"
3735 [(set_attr "type" "mla")]
3736 )
3737
3738 (define_insn "*msub<mode>"
3739 [(set (match_operand:GPI 0 "register_operand" "=r")
3740 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3741 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3742 (match_operand:GPI 2 "register_operand" "r"))))]
3743
3744 ""
3745 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3746 [(set_attr "type" "mla")]
3747 )
3748
3749 ;; zero_extend version of above
3750 (define_insn "*msubsi_uxtw"
3751 [(set (match_operand:DI 0 "register_operand" "=r")
3752 (zero_extend:DI
3753 (minus:SI (match_operand:SI 3 "register_operand" "r")
3754 (mult:SI (match_operand:SI 1 "register_operand" "r")
3755 (match_operand:SI 2 "register_operand" "r")))))]
3756
3757 ""
3758 "msub\\t%w0, %w1, %w2, %w3"
3759 [(set_attr "type" "mla")]
3760 )
3761
3762 (define_insn "*mul<mode>_neg"
3763 [(set (match_operand:GPI 0 "register_operand" "=r")
3764 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3765 (match_operand:GPI 2 "register_operand" "r")))]
3766
3767 ""
3768 "mneg\\t%<w>0, %<w>1, %<w>2"
3769 [(set_attr "type" "mul")]
3770 )
3771
3772 ;; zero_extend version of above
3773 (define_insn "*mulsi_neg_uxtw"
3774 [(set (match_operand:DI 0 "register_operand" "=r")
3775 (zero_extend:DI
3776 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3777 (match_operand:SI 2 "register_operand" "r"))))]
3778
3779 ""
3780 "mneg\\t%w0, %w1, %w2"
3781 [(set_attr "type" "mul")]
3782 )
3783
3784 (define_insn "<su_optab>mulsidi3"
3785 [(set (match_operand:DI 0 "register_operand" "=r")
3786 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3787 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3788 ""
3789 "<su>mull\\t%0, %w1, %w2"
3790 [(set_attr "type" "<su>mull")]
3791 )
3792
3793 (define_insn "<su_optab>maddsidi4"
3794 [(set (match_operand:DI 0 "register_operand" "=r")
3795 (plus:DI (mult:DI
3796 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3797 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3798 (match_operand:DI 3 "register_operand" "r")))]
3799 ""
3800 "<su>maddl\\t%0, %w1, %w2, %3"
3801 [(set_attr "type" "<su>mlal")]
3802 )
3803
3804 (define_insn "<su_optab>msubsidi4"
3805 [(set (match_operand:DI 0 "register_operand" "=r")
3806 (minus:DI
3807 (match_operand:DI 3 "register_operand" "r")
3808 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3809 (ANY_EXTEND:DI
3810 (match_operand:SI 2 "register_operand" "r")))))]
3811 ""
3812 "<su>msubl\\t%0, %w1, %w2, %3"
3813 [(set_attr "type" "<su>mlal")]
3814 )
3815
3816 (define_insn "*<su_optab>mulsidi_neg"
3817 [(set (match_operand:DI 0 "register_operand" "=r")
3818 (mult:DI (neg:DI
3819 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3820 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3821 ""
3822 "<su>mnegl\\t%0, %w1, %w2"
3823 [(set_attr "type" "<su>mull")]
3824 )
3825
3826 (define_expand "<su_optab>mulditi3"
3827 [(set (match_operand:TI 0 "register_operand")
3828 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3829 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3830 ""
3831 {
3832 rtx low = gen_reg_rtx (DImode);
3833 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3834
3835 rtx high = gen_reg_rtx (DImode);
3836 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3837
3838 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3839 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3840 DONE;
3841 })
3842
3843 ;; The default expansion of multi3 using umuldi3_highpart will perform
3844 ;; the additions in an order that fails to combine into two madd insns.
3845 (define_expand "multi3"
3846 [(set (match_operand:TI 0 "register_operand")
3847 (mult:TI (match_operand:TI 1 "register_operand")
3848 (match_operand:TI 2 "register_operand")))]
3849 ""
3850 {
3851 rtx l0 = gen_reg_rtx (DImode);
3852 rtx l1 = gen_lowpart (DImode, operands[1]);
3853 rtx l2 = gen_lowpart (DImode, operands[2]);
3854 rtx h0 = gen_reg_rtx (DImode);
3855 rtx h1 = gen_highpart (DImode, operands[1]);
3856 rtx h2 = gen_highpart (DImode, operands[2]);
3857
3858 emit_insn (gen_muldi3 (l0, l1, l2));
3859 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3860 emit_insn (gen_madddi (h0, h1, l2, h0));
3861 emit_insn (gen_madddi (h0, l1, h2, h0));
3862
3863 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3864 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3865 DONE;
3866 })
3867
3868 (define_insn "<su>muldi3_highpart"
3869 [(set (match_operand:DI 0 "register_operand" "=r")
3870 (truncate:DI
3871 (lshiftrt:TI
3872 (mult:TI
3873 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3874 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3875 (const_int 64))))]
3876 ""
3877 "<su>mulh\\t%0, %1, %2"
3878 [(set_attr "type" "<su>mull")]
3879 )
3880
3881 (define_insn "<su_optab>div<mode>3"
3882 [(set (match_operand:GPI 0 "register_operand" "=r")
3883 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3884 (match_operand:GPI 2 "register_operand" "r")))]
3885 ""
3886 "<su>div\\t%<w>0, %<w>1, %<w>2"
3887 [(set_attr "type" "<su>div")]
3888 )
3889
3890 ;; zero_extend version of above
3891 (define_insn "*<su_optab>divsi3_uxtw"
3892 [(set (match_operand:DI 0 "register_operand" "=r")
3893 (zero_extend:DI
3894 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3895 (match_operand:SI 2 "register_operand" "r"))))]
3896 ""
3897 "<su>div\\t%w0, %w1, %w2"
3898 [(set_attr "type" "<su>div")]
3899 )
3900
3901 ;; -------------------------------------------------------------------
3902 ;; Comparison insns
3903 ;; -------------------------------------------------------------------
3904
3905 (define_insn "cmp<mode>"
3906 [(set (reg:CC CC_REGNUM)
3907 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3908 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3909 ""
3910 "@
3911 cmp\\t%<w>0, %<w>1
3912 cmp\\t%<w>0, %1
3913 cmn\\t%<w>0, #%n1"
3914 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3915 )
3916
3917 (define_insn "fcmp<mode>"
3918 [(set (reg:CCFP CC_REGNUM)
3919 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3920 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3921 "TARGET_FLOAT"
3922 "@
3923 fcmp\\t%<s>0, #0.0
3924 fcmp\\t%<s>0, %<s>1"
3925 [(set_attr "type" "fcmp<s>")]
3926 )
3927
3928 (define_insn "fcmpe<mode>"
3929 [(set (reg:CCFPE CC_REGNUM)
3930 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3931 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3932 "TARGET_FLOAT"
3933 "@
3934 fcmpe\\t%<s>0, #0.0
3935 fcmpe\\t%<s>0, %<s>1"
3936 [(set_attr "type" "fcmp<s>")]
3937 )
3938
3939 (define_insn "*cmp_swp_<shift>_reg<mode>"
3940 [(set (reg:CC_SWP CC_REGNUM)
3941 (compare:CC_SWP (ASHIFT:GPI
3942 (match_operand:GPI 0 "register_operand" "r")
3943 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3944 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3945 ""
3946 "cmp\\t%<w>2, %<w>0, <shift> %1"
3947 [(set_attr "type" "alus_shift_imm")]
3948 )
3949
3950 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3951 [(set (reg:CC_SWP CC_REGNUM)
3952 (compare:CC_SWP (ANY_EXTEND:GPI
3953 (match_operand:ALLX 0 "register_operand" "r"))
3954 (match_operand:GPI 1 "register_operand" "r")))]
3955 ""
3956 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3957 [(set_attr "type" "alus_ext")]
3958 )
3959
3960 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3961 [(set (reg:CC_SWP CC_REGNUM)
3962 (compare:CC_SWP (ashift:GPI
3963 (ANY_EXTEND:GPI
3964 (match_operand:ALLX 0 "register_operand" "r"))
3965 (match_operand 1 "aarch64_imm3" "Ui3"))
3966 (match_operand:GPI 2 "register_operand" "r")))]
3967 ""
3968 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3969 [(set_attr "type" "alus_ext")]
3970 )
3971
3972 ;; -------------------------------------------------------------------
3973 ;; Store-flag and conditional select insns
3974 ;; -------------------------------------------------------------------
3975
3976 (define_expand "cstore<mode>4"
3977 [(set (match_operand:SI 0 "register_operand")
3978 (match_operator:SI 1 "aarch64_comparison_operator"
3979 [(match_operand:GPI 2 "register_operand")
3980 (match_operand:GPI 3 "aarch64_plus_operand")]))]
3981 ""
3982 "
3983 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3984 operands[3]);
3985 operands[3] = const0_rtx;
3986 "
3987 )
3988
3989 (define_expand "cstorecc4"
3990 [(set (match_operand:SI 0 "register_operand")
3991 (match_operator 1 "aarch64_comparison_operator_mode"
3992 [(match_operand 2 "cc_register")
3993 (match_operand 3 "const0_operand")]))]
3994 ""
3995 "{
3996 emit_insn (gen_rtx_SET (operands[0], operands[1]));
3997 DONE;
3998 }")
3999
4000
4001 (define_expand "cstore<mode>4"
4002 [(set (match_operand:SI 0 "register_operand")
4003 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4004 [(match_operand:GPF 2 "register_operand")
4005 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
4006 ""
4007 "
4008 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4009 operands[3]);
4010 operands[3] = const0_rtx;
4011 "
4012 )
4013
4014 (define_insn "aarch64_cstore<mode>"
4015 [(set (match_operand:ALLI 0 "register_operand" "=r")
4016 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4017 [(match_operand 2 "cc_register" "") (const_int 0)]))]
4018 ""
4019 "cset\\t%<w>0, %m1"
4020 [(set_attr "type" "csel")]
4021 )
4022
4023 ;; For a 24-bit immediate CST we can optimize the compare for equality
4024 ;; and branch sequence from:
4025 ;; mov x0, #imm1
4026 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
4027 ;; cmp x1, x0
4028 ;; cset x2, <ne,eq>
4029 ;; into the shorter:
4030 ;; sub x0, x1, #(CST & 0xfff000)
4031 ;; subs x0, x0, #(CST & 0x000fff)
4032 ;; cset x2, <ne, eq>.
4033 (define_insn_and_split "*compare_cstore<mode>_insn"
4034 [(set (match_operand:GPI 0 "register_operand" "=r")
4035 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
4036 (match_operand:GPI 2 "aarch64_imm24" "n")))
4037 (clobber (reg:CC CC_REGNUM))]
4038 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4039 && !aarch64_plus_operand (operands[2], <MODE>mode)
4040 && !reload_completed"
4041 "#"
4042 "&& true"
4043 [(const_int 0)]
4044 {
4045 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4046 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4047 rtx tmp = gen_reg_rtx (<MODE>mode);
4048 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4049 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4050 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4051 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4052 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4053 DONE;
4054 }
4055 [(set_attr "type" "csel")]
4056 )
4057
4058 ;; zero_extend version of the above
4059 (define_insn "*cstoresi_insn_uxtw"
4060 [(set (match_operand:DI 0 "register_operand" "=r")
4061 (zero_extend:DI
4062 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4063 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4064 ""
4065 "cset\\t%w0, %m1"
4066 [(set_attr "type" "csel")]
4067 )
4068
4069 (define_insn "cstore<mode>_neg"
4070 [(set (match_operand:ALLI 0 "register_operand" "=r")
4071 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4072 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4073 ""
4074 "csetm\\t%<w>0, %m1"
4075 [(set_attr "type" "csel")]
4076 )
4077
4078 ;; zero_extend version of the above
4079 (define_insn "*cstoresi_neg_uxtw"
4080 [(set (match_operand:DI 0 "register_operand" "=r")
4081 (zero_extend:DI
4082 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4083 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4084 ""
4085 "csetm\\t%w0, %m1"
4086 [(set_attr "type" "csel")]
4087 )
4088
4089 (define_expand "cmov<mode>6"
4090 [(set (match_operand:GPI 0 "register_operand")
4091 (if_then_else:GPI
4092 (match_operator 1 "aarch64_comparison_operator"
4093 [(match_operand:GPI 2 "register_operand")
4094 (match_operand:GPI 3 "aarch64_plus_operand")])
4095 (match_operand:GPI 4 "register_operand")
4096 (match_operand:GPI 5 "register_operand")))]
4097 ""
4098 "
4099 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4100 operands[3]);
4101 operands[3] = const0_rtx;
4102 "
4103 )
4104
4105 (define_expand "cmov<mode>6"
4106 [(set (match_operand:GPF 0 "register_operand")
4107 (if_then_else:GPF
4108 (match_operator 1 "aarch64_comparison_operator"
4109 [(match_operand:GPF 2 "register_operand")
4110 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4111 (match_operand:GPF 4 "register_operand")
4112 (match_operand:GPF 5 "register_operand")))]
4113 ""
4114 "
4115 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4116 operands[3]);
4117 operands[3] = const0_rtx;
4118 "
4119 )
4120
4121 (define_insn "*cmov<mode>_insn"
4122 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4123 (if_then_else:ALLI
4124 (match_operator 1 "aarch64_comparison_operator"
4125 [(match_operand 2 "cc_register" "") (const_int 0)])
4126 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4127 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4128 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4129 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4130 ;; Final two alternatives should be unreachable, but included for completeness
4131 "@
4132 csel\\t%<w>0, %<w>3, %<w>4, %m1
4133 csinv\\t%<w>0, %<w>3, <w>zr, %m1
4134 csinv\\t%<w>0, %<w>4, <w>zr, %M1
4135 csinc\\t%<w>0, %<w>3, <w>zr, %m1
4136 csinc\\t%<w>0, %<w>4, <w>zr, %M1
4137 mov\\t%<w>0, -1
4138 mov\\t%<w>0, 1"
4139 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4140 )
4141
4142 ;; zero_extend version of above
4143 (define_insn "*cmovsi_insn_uxtw"
4144 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4145 (zero_extend:DI
4146 (if_then_else:SI
4147 (match_operator 1 "aarch64_comparison_operator"
4148 [(match_operand 2 "cc_register" "") (const_int 0)])
4149 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4150 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4151 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4152 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4153 ;; Final two alternatives should be unreachable, but included for completeness
4154 "@
4155 csel\\t%w0, %w3, %w4, %m1
4156 csinv\\t%w0, %w3, wzr, %m1
4157 csinv\\t%w0, %w4, wzr, %M1
4158 csinc\\t%w0, %w3, wzr, %m1
4159 csinc\\t%w0, %w4, wzr, %M1
4160 mov\\t%w0, -1
4161 mov\\t%w0, 1"
4162 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4163 )
4164
4165 (define_insn "*cmovdi_insn_uxtw"
4166 [(set (match_operand:DI 0 "register_operand" "=r")
4167 (if_then_else:DI
4168 (match_operator 1 "aarch64_comparison_operator"
4169 [(match_operand 2 "cc_register" "") (const_int 0)])
4170 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4171 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4172 ""
4173 "csel\\t%w0, %w3, %w4, %m1"
4174 [(set_attr "type" "csel")]
4175 )
4176
4177 (define_insn "*cmov<mode>_insn"
4178 [(set (match_operand:GPF 0 "register_operand" "=w")
4179 (if_then_else:GPF
4180 (match_operator 1 "aarch64_comparison_operator"
4181 [(match_operand 2 "cc_register" "") (const_int 0)])
4182 (match_operand:GPF 3 "register_operand" "w")
4183 (match_operand:GPF 4 "register_operand" "w")))]
4184 "TARGET_FLOAT"
4185 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4186 [(set_attr "type" "fcsel")]
4187 )
4188
4189 (define_expand "mov<mode>cc"
4190 [(set (match_operand:ALLI 0 "register_operand")
4191 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4192 (match_operand:ALLI 2 "register_operand")
4193 (match_operand:ALLI 3 "register_operand")))]
4194 ""
4195 {
4196 rtx ccreg;
4197 enum rtx_code code = GET_CODE (operands[1]);
4198
4199 if (code == UNEQ || code == LTGT)
4200 FAIL;
4201
4202 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4203 XEXP (operands[1], 1));
4204 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4205 }
4206 )
4207
4208 (define_expand "mov<GPF:mode><GPI:mode>cc"
4209 [(set (match_operand:GPI 0 "register_operand")
4210 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4211 (match_operand:GPF 2 "register_operand")
4212 (match_operand:GPF 3 "register_operand")))]
4213 ""
4214 {
4215 rtx ccreg;
4216 enum rtx_code code = GET_CODE (operands[1]);
4217
4218 if (code == UNEQ || code == LTGT)
4219 FAIL;
4220
4221 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4222 XEXP (operands[1], 1));
4223 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4224 }
4225 )
4226
4227 (define_expand "mov<mode>cc"
4228 [(set (match_operand:GPF 0 "register_operand")
4229 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4230 (match_operand:GPF 2 "register_operand")
4231 (match_operand:GPF 3 "register_operand")))]
4232 ""
4233 {
4234 rtx ccreg;
4235 enum rtx_code code = GET_CODE (operands[1]);
4236
4237 if (code == UNEQ || code == LTGT)
4238 FAIL;
4239
4240 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4241 XEXP (operands[1], 1));
4242 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4243 }
4244 )
4245
4246 (define_expand "<neg_not_op><mode>cc"
4247 [(set (match_operand:GPI 0 "register_operand")
4248 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4249 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4250 (match_operand:GPI 3 "register_operand")))]
4251 ""
4252 {
4253 rtx ccreg;
4254 enum rtx_code code = GET_CODE (operands[1]);
4255
4256 if (code == UNEQ || code == LTGT)
4257 FAIL;
4258
4259 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4260 XEXP (operands[1], 1));
4261 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4262 }
4263 )
4264
4265 ;; CRC32 instructions.
4266 (define_insn "aarch64_<crc_variant>"
4267 [(set (match_operand:SI 0 "register_operand" "=r")
4268 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4269 (match_operand:<crc_mode> 2 "register_operand" "r")]
4270 CRC))]
4271 "TARGET_CRC32"
4272 {
4273 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4274 return "<crc_variant>\\t%w0, %w1, %x2";
4275 else
4276 return "<crc_variant>\\t%w0, %w1, %w2";
4277 }
4278 [(set_attr "type" "crc")]
4279 )
4280
4281 (define_insn "*csinc2<mode>_insn"
4282 [(set (match_operand:GPI 0 "register_operand" "=r")
4283 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4284 (match_operand:GPI 1 "register_operand" "r")))]
4285 ""
4286 "cinc\\t%<w>0, %<w>1, %m2"
4287 [(set_attr "type" "csel")]
4288 )
4289
4290 (define_insn "csinc3<mode>_insn"
4291 [(set (match_operand:GPI 0 "register_operand" "=r")
4292 (if_then_else:GPI
4293 (match_operand 1 "aarch64_comparison_operation" "")
4294 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4295 (const_int 1))
4296 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4297 ""
4298 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4299 [(set_attr "type" "csel")]
4300 )
4301
4302 (define_insn "*csinv3<mode>_insn"
4303 [(set (match_operand:GPI 0 "register_operand" "=r")
4304 (if_then_else:GPI
4305 (match_operand 1 "aarch64_comparison_operation" "")
4306 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4307 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4308 ""
4309 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4310 [(set_attr "type" "csel")]
4311 )
4312
4313 (define_insn "csneg3_uxtw_insn"
4314 [(set (match_operand:DI 0 "register_operand" "=r")
4315 (zero_extend:DI
4316 (if_then_else:SI
4317 (match_operand 1 "aarch64_comparison_operation" "")
4318 (neg:SI (match_operand:SI 2 "register_operand" "r"))
4319 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4320 ""
4321 "csneg\\t%w0, %w3, %w2, %M1"
4322 [(set_attr "type" "csel")]
4323 )
4324
4325 (define_insn "csneg3<mode>_insn"
4326 [(set (match_operand:GPI 0 "register_operand" "=r")
4327 (if_then_else:GPI
4328 (match_operand 1 "aarch64_comparison_operation" "")
4329 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4330 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4331 ""
4332 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4333 [(set_attr "type" "csel")]
4334 )
4335
4336 ;; If X can be loaded by a single CNT[BHWD] instruction,
4337 ;;
4338 ;; A = UMAX (B, X)
4339 ;;
4340 ;; is equivalent to:
4341 ;;
4342 ;; TMP = UQDEC[BHWD] (B, X)
4343 ;; A = TMP + X
4344 ;;
4345 ;; Defining the pattern this way means that:
4346 ;;
4347 ;; A = UMAX (B, X) - X
4348 ;;
4349 ;; becomes:
4350 ;;
4351 ;; TMP1 = UQDEC[BHWD] (B, X)
4352 ;; TMP2 = TMP1 + X
4353 ;; A = TMP2 - X
4354 ;;
4355 ;; which combine can optimize to:
4356 ;;
4357 ;; A = UQDEC[BHWD] (B, X)
4358 ;;
4359 ;; We don't use match_operand predicates because the order of the operands
4360 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4361 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4362 (define_expand "umax<mode>3"
4363 [(set (match_operand:GPI 0 "register_operand")
4364 (umax:GPI (match_operand:GPI 1 "")
4365 (match_operand:GPI 2 "")))]
4366 "TARGET_SVE"
4367 {
4368 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4369 std::swap (operands[1], operands[2]);
4370 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4371 FAIL;
4372 rtx temp = gen_reg_rtx (<MODE>mode);
4373 operands[1] = force_reg (<MODE>mode, operands[1]);
4374 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4375 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4376 DONE;
4377 }
4378 )
4379
4380 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4381 (define_insn "aarch64_uqdec<mode>"
4382 [(set (match_operand:GPI 0 "register_operand" "=r")
4383 (minus:GPI
4384 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4385 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4386 (match_dup 2)))]
4387 "TARGET_SVE"
4388 {
4389 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4390 }
4391 )
4392
4393 ;; -------------------------------------------------------------------
4394 ;; Logical operations
4395 ;; -------------------------------------------------------------------
4396
4397
4398 (define_insn_and_split "*aarch64_and<mode>_imm2"
4399 [(set (match_operand:GPI 0 "register_operand" "=rk")
4400 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4401 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4402 ""
4403 "#"
4404 "true"
4405 [(const_int 0)]
4406 {
4407 HOST_WIDE_INT val = INTVAL (operands[2]);
4408 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4409 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4410
4411 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4412 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4413 DONE;
4414 }
4415 )
4416
4417 (define_insn "<optab><mode>3"
4418 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4419 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4420 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4421 ""
4422 "@
4423 <logical>\\t%<w>0, %<w>1, %<w>2
4424 <logical>\\t%<w>0, %<w>1, %2
4425 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4426 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4427 (set_attr "arch" "*,*,simd")]
4428 )
4429
4430 ;; zero_extend version of above
4431 (define_insn "*<optab>si3_uxtw"
4432 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4433 (zero_extend:DI
4434 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4435 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4436 ""
4437 "@
4438 <logical>\\t%w0, %w1, %w2
4439 <logical>\\t%w0, %w1, %2"
4440 [(set_attr "type" "logic_reg,logic_imm")]
4441 )
4442
4443 (define_insn "*and<mode>3_compare0"
4444 [(set (reg:CC_NZ CC_REGNUM)
4445 (compare:CC_NZ
4446 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4447 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4448 (const_int 0)))
4449 (set (match_operand:GPI 0 "register_operand" "=r,r")
4450 (and:GPI (match_dup 1) (match_dup 2)))]
4451 ""
4452 "@
4453 ands\\t%<w>0, %<w>1, %<w>2
4454 ands\\t%<w>0, %<w>1, %2"
4455 [(set_attr "type" "logics_reg,logics_imm")]
4456 )
4457
4458 ;; zero_extend version of above
4459 (define_insn "*andsi3_compare0_uxtw"
4460 [(set (reg:CC_NZ CC_REGNUM)
4461 (compare:CC_NZ
4462 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4463 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4464 (const_int 0)))
4465 (set (match_operand:DI 0 "register_operand" "=r,r")
4466 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4467 ""
4468 "@
4469 ands\\t%w0, %w1, %w2
4470 ands\\t%w0, %w1, %2"
4471 [(set_attr "type" "logics_reg,logics_imm")]
4472 )
4473
4474 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4475 [(set (reg:CC_NZ CC_REGNUM)
4476 (compare:CC_NZ
4477 (and:GPI (SHIFT:GPI
4478 (match_operand:GPI 1 "register_operand" "r")
4479 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4480 (match_operand:GPI 3 "register_operand" "r"))
4481 (const_int 0)))
4482 (set (match_operand:GPI 0 "register_operand" "=r")
4483 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4484 ""
4485 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4486 [(set_attr "type" "logics_shift_imm")]
4487 )
4488
4489 ;; zero_extend version of above
4490 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4491 [(set (reg:CC_NZ CC_REGNUM)
4492 (compare:CC_NZ
4493 (and:SI (SHIFT:SI
4494 (match_operand:SI 1 "register_operand" "r")
4495 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4496 (match_operand:SI 3 "register_operand" "r"))
4497 (const_int 0)))
4498 (set (match_operand:DI 0 "register_operand" "=r")
4499 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4500 (match_dup 3))))]
4501 ""
4502 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4503 [(set_attr "type" "logics_shift_imm")]
4504 )
4505
4506 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4507 [(set (match_operand:GPI 0 "register_operand" "=r")
4508 (LOGICAL:GPI (SHIFT:GPI
4509 (match_operand:GPI 1 "register_operand" "r")
4510 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4511 (match_operand:GPI 3 "register_operand" "r")))]
4512 ""
4513 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4514 [(set_attr "type" "logic_shift_imm")]
4515 )
4516
4517 (define_insn "*<optab>_rol<mode>3"
4518 [(set (match_operand:GPI 0 "register_operand" "=r")
4519 (LOGICAL:GPI (rotate:GPI
4520 (match_operand:GPI 1 "register_operand" "r")
4521 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4522 (match_operand:GPI 3 "register_operand" "r")))]
4523 ""
4524 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4525 [(set_attr "type" "logic_shift_imm")]
4526 )
4527
4528 ;; zero_extend versions of above
4529 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4530 [(set (match_operand:DI 0 "register_operand" "=r")
4531 (zero_extend:DI
4532 (LOGICAL:SI (SHIFT:SI
4533 (match_operand:SI 1 "register_operand" "r")
4534 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4535 (match_operand:SI 3 "register_operand" "r"))))]
4536 ""
4537 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4538 [(set_attr "type" "logic_shift_imm")]
4539 )
4540
4541 (define_insn "*<optab>_rolsi3_uxtw"
4542 [(set (match_operand:DI 0 "register_operand" "=r")
4543 (zero_extend:DI
4544 (LOGICAL:SI (rotate:SI
4545 (match_operand:SI 1 "register_operand" "r")
4546 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4547 (match_operand:SI 3 "register_operand" "r"))))]
4548 ""
4549 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4550 [(set_attr "type" "logic_shift_imm")]
4551 )
4552
4553 (define_insn "one_cmpl<mode>2"
4554 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4555 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4556 ""
4557 "@
4558 mvn\\t%<w>0, %<w>1
4559 mvn\\t%0.8b, %1.8b"
4560 [(set_attr "type" "logic_reg,neon_logic")
4561 (set_attr "arch" "*,simd")]
4562 )
4563
4564 (define_insn "*one_cmpl_<optab><mode>2"
4565 [(set (match_operand:GPI 0 "register_operand" "=r")
4566 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4567 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4568 ""
4569 "mvn\\t%<w>0, %<w>1, <shift> %2"
4570 [(set_attr "type" "logic_shift_imm")]
4571 )
4572
4573 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4574
4575 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4576 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4577 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4578 (match_operand:GPI 2 "register_operand" "r,w")))]
4579 ""
4580 "@
4581 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4582 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4583 [(set_attr "type" "logic_reg,neon_logic")
4584 (set_attr "arch" "*,simd")]
4585 )
4586
4587 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4588 [(set (match_operand:DI 0 "register_operand" "=r")
4589 (zero_extend:DI
4590 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4591 (match_operand:SI 2 "register_operand" "r"))))]
4592 ""
4593 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4594 [(set_attr "type" "logic_reg")]
4595 )
4596
4597 (define_insn "*xor_one_cmplsidi3_ze"
4598 [(set (match_operand:DI 0 "register_operand" "=r")
4599 (zero_extend:DI
4600 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4601 (match_operand:SI 2 "register_operand" "r")))))]
4602 ""
4603 "eon\\t%w0, %w1, %w2"
4604 [(set_attr "type" "logic_reg")]
4605 )
4606
4607 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4608 ;; eon does not operate on SIMD registers so the vector variant must be split.
4609 (define_insn_and_split "*xor_one_cmpl<mode>3"
4610 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4611 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4612 (match_operand:GPI 2 "register_operand" "r,w"))))]
4613 ""
4614 "@
4615 eon\\t%<w>0, %<w>1, %<w>2
4616 #"
4617 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4618 [(set (match_operand:GPI 0 "register_operand" "=w")
4619 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4620 (match_operand:GPI 2 "register_operand" "w")))
4621 (set (match_dup 0) (not:GPI (match_dup 0)))]
4622 ""
4623 [(set_attr "type" "logic_reg,multiple")
4624 (set_attr "arch" "*,simd")]
4625 )
4626
4627 (define_insn "*and_one_cmpl<mode>3_compare0"
4628 [(set (reg:CC_NZ CC_REGNUM)
4629 (compare:CC_NZ
4630 (and:GPI (not:GPI
4631 (match_operand:GPI 1 "register_operand" "r"))
4632 (match_operand:GPI 2 "register_operand" "r"))
4633 (const_int 0)))
4634 (set (match_operand:GPI 0 "register_operand" "=r")
4635 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4636 ""
4637 "bics\\t%<w>0, %<w>2, %<w>1"
4638 [(set_attr "type" "logics_reg")]
4639 )
4640
4641 ;; zero_extend version of above
4642 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4643 [(set (reg:CC_NZ CC_REGNUM)
4644 (compare:CC_NZ
4645 (and:SI (not:SI
4646 (match_operand:SI 1 "register_operand" "r"))
4647 (match_operand:SI 2 "register_operand" "r"))
4648 (const_int 0)))
4649 (set (match_operand:DI 0 "register_operand" "=r")
4650 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4651 ""
4652 "bics\\t%w0, %w2, %w1"
4653 [(set_attr "type" "logics_reg")]
4654 )
4655
4656 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4657 [(set (reg:CC_NZ CC_REGNUM)
4658 (compare:CC_NZ
4659 (and:GPI (not:GPI
4660 (match_operand:GPI 0 "register_operand" "r"))
4661 (match_operand:GPI 1 "register_operand" "r"))
4662 (const_int 0)))]
4663 ""
4664 "bics\\t<w>zr, %<w>1, %<w>0"
4665 [(set_attr "type" "logics_reg")]
4666 )
4667
4668 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4669 [(set (match_operand:GPI 0 "register_operand" "=r")
4670 (LOGICAL:GPI (not:GPI
4671 (SHIFT:GPI
4672 (match_operand:GPI 1 "register_operand" "r")
4673 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4674 (match_operand:GPI 3 "register_operand" "r")))]
4675 ""
4676 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4677 [(set_attr "type" "logic_shift_imm")]
4678 )
4679
4680 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4681 [(set (match_operand:GPI 0 "register_operand" "=r")
4682 (not:GPI (xor:GPI
4683 (SHIFT:GPI
4684 (match_operand:GPI 1 "register_operand" "r")
4685 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4686 (match_operand:GPI 3 "register_operand" "r"))))]
4687 ""
4688 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4689 [(set_attr "type" "logic_shift_imm")]
4690 )
4691
4692 ;; Zero-extend version of the above.
4693 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4694 [(set (match_operand:DI 0 "register_operand" "=r")
4695 (zero_extend:DI
4696 (not:SI (xor:SI
4697 (SHIFT:SI
4698 (match_operand:SI 1 "register_operand" "r")
4699 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4700 (match_operand:SI 3 "register_operand" "r")))))]
4701 ""
4702 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4703 [(set_attr "type" "logic_shift_imm")]
4704 )
4705
4706 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4707 [(set (reg:CC_NZ CC_REGNUM)
4708 (compare:CC_NZ
4709 (and:GPI (not:GPI
4710 (SHIFT:GPI
4711 (match_operand:GPI 1 "register_operand" "r")
4712 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4713 (match_operand:GPI 3 "register_operand" "r"))
4714 (const_int 0)))
4715 (set (match_operand:GPI 0 "register_operand" "=r")
4716 (and:GPI (not:GPI
4717 (SHIFT:GPI
4718 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4719 ""
4720 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4721 [(set_attr "type" "logics_shift_imm")]
4722 )
4723
4724 ;; zero_extend version of above
4725 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4726 [(set (reg:CC_NZ CC_REGNUM)
4727 (compare:CC_NZ
4728 (and:SI (not:SI
4729 (SHIFT:SI
4730 (match_operand:SI 1 "register_operand" "r")
4731 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4732 (match_operand:SI 3 "register_operand" "r"))
4733 (const_int 0)))
4734 (set (match_operand:DI 0 "register_operand" "=r")
4735 (zero_extend:DI (and:SI
4736 (not:SI
4737 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4738 ""
4739 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4740 [(set_attr "type" "logics_shift_imm")]
4741 )
4742
4743 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4744 [(set (reg:CC_NZ CC_REGNUM)
4745 (compare:CC_NZ
4746 (and:GPI (not:GPI
4747 (SHIFT:GPI
4748 (match_operand:GPI 0 "register_operand" "r")
4749 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4750 (match_operand:GPI 2 "register_operand" "r"))
4751 (const_int 0)))]
4752 ""
4753 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4754 [(set_attr "type" "logics_shift_imm")]
4755 )
4756
4757 (define_insn "clz<mode>2"
4758 [(set (match_operand:GPI 0 "register_operand" "=r")
4759 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4760 ""
4761 "clz\\t%<w>0, %<w>1"
4762 [(set_attr "type" "clz")]
4763 )
4764
4765 (define_expand "ffs<mode>2"
4766 [(match_operand:GPI 0 "register_operand")
4767 (match_operand:GPI 1 "register_operand")]
4768 ""
4769 {
4770 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4771 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4772
4773 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4774 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4775 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4776 DONE;
4777 }
4778 )
4779
4780 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4781 ;;
4782 ;; MOV v.1d, x0
4783 ;; CNT v1.8b, v.8b
4784 ;; ADDV b2, v1.8b
4785 ;; MOV w0, v2.b[0]
4786
4787 (define_expand "popcount<mode>2"
4788 [(match_operand:GPI 0 "register_operand")
4789 (match_operand:GPI 1 "register_operand")]
4790 "TARGET_SIMD"
4791 {
4792 rtx v = gen_reg_rtx (V8QImode);
4793 rtx v1 = gen_reg_rtx (V8QImode);
4794 rtx r = gen_reg_rtx (QImode);
4795 rtx in = operands[1];
4796 rtx out = operands[0];
4797 if(<MODE>mode == SImode)
4798 {
4799 rtx tmp;
4800 tmp = gen_reg_rtx (DImode);
4801 /* If we have SImode, zero extend to DImode, pop count does
4802 not change if we have extra zeros. */
4803 emit_insn (gen_zero_extendsidi2 (tmp, in));
4804 in = tmp;
4805 }
4806 emit_move_insn (v, gen_lowpart (V8QImode, in));
4807 emit_insn (gen_popcountv8qi2 (v1, v));
4808 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4809 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4810 DONE;
4811 })
4812
4813 (define_insn "clrsb<mode>2"
4814 [(set (match_operand:GPI 0 "register_operand" "=r")
4815 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4816 ""
4817 "cls\\t%<w>0, %<w>1"
4818 [(set_attr "type" "clz")]
4819 )
4820
4821 (define_insn "rbit<mode>2"
4822 [(set (match_operand:GPI 0 "register_operand" "=r")
4823 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4824 ""
4825 "rbit\\t%<w>0, %<w>1"
4826 [(set_attr "type" "rbit")]
4827 )
4828
4829 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4830 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4831 ;; expression and split after reload to enable scheduling them apart if
4832 ;; needed.
4833
4834 (define_insn_and_split "ctz<mode>2"
4835 [(set (match_operand:GPI 0 "register_operand" "=r")
4836 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4837 ""
4838 "#"
4839 "reload_completed"
4840 [(const_int 0)]
4841 "
4842 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4843 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4844 DONE;
4845 ")
4846
4847 (define_insn "*and<mode>_compare0"
4848 [(set (reg:CC_NZ CC_REGNUM)
4849 (compare:CC_NZ
4850 (match_operand:SHORT 0 "register_operand" "r")
4851 (const_int 0)))]
4852 ""
4853 "tst\\t%<w>0, <short_mask>"
4854 [(set_attr "type" "alus_imm")]
4855 )
4856
4857 (define_insn "*ands<GPI:mode>_compare0"
4858 [(set (reg:CC_NZ CC_REGNUM)
4859 (compare:CC_NZ
4860 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4861 (const_int 0)))
4862 (set (match_operand:GPI 0 "register_operand" "=r")
4863 (zero_extend:GPI (match_dup 1)))]
4864 ""
4865 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4866 [(set_attr "type" "alus_imm")]
4867 )
4868
4869 (define_insn "*and<mode>3nr_compare0"
4870 [(set (reg:CC_NZ CC_REGNUM)
4871 (compare:CC_NZ
4872 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4873 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4874 (const_int 0)))]
4875 ""
4876 "@
4877 tst\\t%<w>0, %<w>1
4878 tst\\t%<w>0, %1"
4879 [(set_attr "type" "logics_reg,logics_imm")]
4880 )
4881
4882 (define_split
4883 [(set (reg:CC_NZ CC_REGNUM)
4884 (compare:CC_NZ
4885 (and:GPI (match_operand:GPI 0 "register_operand")
4886 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4887 (const_int 0)))
4888 (clobber (match_operand:SI 2 "register_operand"))]
4889 ""
4890 [(set (match_dup 2) (match_dup 1))
4891 (set (reg:CC_NZ CC_REGNUM)
4892 (compare:CC_NZ
4893 (and:GPI (match_dup 0)
4894 (match_dup 2))
4895 (const_int 0)))]
4896 )
4897
4898 (define_insn "*and<mode>3nr_compare0_zextract"
4899 [(set (reg:CC_NZ CC_REGNUM)
4900 (compare:CC_NZ
4901 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4902 (match_operand:GPI 1 "const_int_operand" "n")
4903 (match_operand:GPI 2 "const_int_operand" "n"))
4904 (const_int 0)))]
4905 "INTVAL (operands[1]) > 0
4906 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4907 <= GET_MODE_BITSIZE (<MODE>mode))
4908 && aarch64_bitmask_imm (
4909 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4910 operands[2])),
4911 <MODE>mode)"
4912 {
4913 operands[1]
4914 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4915 return "tst\\t%<w>0, %1";
4916 }
4917 [(set_attr "type" "logics_shift_imm")]
4918 )
4919
4920 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4921 [(set (reg:CC_NZ CC_REGNUM)
4922 (compare:CC_NZ
4923 (and:GPI (SHIFT:GPI
4924 (match_operand:GPI 0 "register_operand" "r")
4925 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4926 (match_operand:GPI 2 "register_operand" "r"))
4927 (const_int 0)))]
4928 ""
4929 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4930 [(set_attr "type" "logics_shift_imm")]
4931 )
4932
4933 (define_split
4934 [(set (reg:CC_NZ CC_REGNUM)
4935 (compare:CC_NZ
4936 (and:GPI (SHIFT:GPI
4937 (match_operand:GPI 0 "register_operand")
4938 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4939 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4940 (const_int 0)))
4941 (clobber (match_operand:SI 3 "register_operand"))]
4942 ""
4943 [(set (match_dup 3) (match_dup 2))
4944 (set (reg:CC_NZ CC_REGNUM)
4945 (compare:CC_NZ
4946 (and:GPI (SHIFT:GPI
4947 (match_dup 0)
4948 (match_dup 1))
4949 (match_dup 3))
4950 (const_int 0)))]
4951 )
4952
4953 ;; -------------------------------------------------------------------
4954 ;; Shifts
4955 ;; -------------------------------------------------------------------
4956
4957 (define_expand "<optab><mode>3"
4958 [(set (match_operand:GPI 0 "register_operand")
4959 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4960 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4961 ""
4962 {
4963 if (CONST_INT_P (operands[2]))
4964 {
4965 operands[2] = GEN_INT (INTVAL (operands[2])
4966 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4967
4968 if (operands[2] == const0_rtx)
4969 {
4970 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4971 DONE;
4972 }
4973 }
4974 }
4975 )
4976
4977 (define_expand "ashl<mode>3"
4978 [(set (match_operand:SHORT 0 "register_operand")
4979 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4980 (match_operand:QI 2 "const_int_operand")))]
4981 ""
4982 {
4983 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4984
4985 if (operands[2] == const0_rtx)
4986 {
4987 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4988 DONE;
4989 }
4990 }
4991 )
4992
4993 (define_expand "rotr<mode>3"
4994 [(set (match_operand:GPI 0 "register_operand")
4995 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4996 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4997 ""
4998 {
4999 if (CONST_INT_P (operands[2]))
5000 {
5001 operands[2] = GEN_INT (INTVAL (operands[2])
5002 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5003
5004 if (operands[2] == const0_rtx)
5005 {
5006 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5007 DONE;
5008 }
5009 }
5010 }
5011 )
5012
5013 (define_expand "rotl<mode>3"
5014 [(set (match_operand:GPI 0 "register_operand")
5015 (rotatert:GPI (match_operand:GPI 1 "register_operand")
5016 (match_operand:QI 2 "aarch64_reg_or_imm")))]
5017 ""
5018 {
5019 /* (SZ - cnt) % SZ == -cnt % SZ */
5020 if (CONST_INT_P (operands[2]))
5021 {
5022 operands[2] = GEN_INT ((-INTVAL (operands[2]))
5023 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5024 if (operands[2] == const0_rtx)
5025 {
5026 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5027 DONE;
5028 }
5029 }
5030 else
5031 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
5032 NULL_RTX, 1);
5033 }
5034 )
5035
5036 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5037 ;; they truncate the shift/rotate amount by the size of the registers they
5038 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
5039 ;; such redundant masking instructions. GCC can do that automatically when
5040 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5041 ;; because some of the SISD shift alternatives don't perform this truncations.
5042 ;; So this pattern exists to catch such cases.
5043
5044 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5045 [(set (match_operand:GPI 0 "register_operand" "=r")
5046 (SHIFT:GPI
5047 (match_operand:GPI 1 "register_operand" "r")
5048 (match_operator 4 "subreg_lowpart_operator"
5049 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5050 (match_operand 3 "const_int_operand" "n"))])))]
5051 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5052 "<shift>\t%<w>0, %<w>1, %<w>2"
5053 [(set_attr "type" "shift_reg")]
5054 )
5055
5056 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5057 [(set (match_operand:GPI 0 "register_operand" "=&r")
5058 (SHIFT:GPI
5059 (match_operand:GPI 1 "register_operand" "r")
5060 (match_operator 4 "subreg_lowpart_operator"
5061 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5062 (match_operand 3 "const_int_operand" "n")))])))]
5063 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5064 "#"
5065 "&& true"
5066 [(const_int 0)]
5067 {
5068 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5069 : lowpart_subreg (SImode, operands[0], <MODE>mode));
5070 emit_insn (gen_negsi2 (tmp, operands[2]));
5071
5072 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5073 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5074 SUBREG_BYTE (operands[4]));
5075 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5076 DONE;
5077 }
5078 )
5079
5080 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5081 [(set (match_operand:GPI 0 "register_operand" "=&r")
5082 (ashift:GPI
5083 (match_operand:GPI 1 "register_operand" "r")
5084 (minus:QI (match_operand 2 "const_int_operand" "n")
5085 (match_operator 5 "subreg_lowpart_operator"
5086 [(and:SI (match_operand:SI 3 "register_operand" "r")
5087 (match_operand 4 "const_int_operand" "n"))]))))]
5088 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5089 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5090 "#"
5091 "&& true"
5092 [(const_int 0)]
5093 {
5094 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5095 : operands[0]);
5096
5097 emit_insn (gen_negsi2 (tmp, operands[3]));
5098
5099 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5100 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5101 SUBREG_BYTE (operands[5]));
5102
5103 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5104 DONE;
5105 }
5106 )
5107
5108 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5109 [(set (match_operand:DI 0 "register_operand" "=r")
5110 (SHIFT:DI
5111 (match_operand:DI 1 "register_operand" "r")
5112 (match_operator 4 "subreg_lowpart_operator"
5113 [(and:SI (match_operand:SI 2 "register_operand" "r")
5114 (match_operand 3 "const_int_operand" "n"))])))]
5115 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5116 {
5117 rtx xop[3];
5118 xop[0] = operands[0];
5119 xop[1] = operands[1];
5120 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5121 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5122 return "";
5123 }
5124 [(set_attr "type" "shift_reg")]
5125 )
5126
5127 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5128 [(set (match_operand:GPI 0 "register_operand" "=&r")
5129 (ASHIFT:GPI
5130 (match_operand:GPI 1 "register_operand" "r")
5131 (minus:QI (match_operand 2 "const_int_operand" "n")
5132 (match_operand:QI 3 "register_operand" "r"))))]
5133 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5134 "#"
5135 "&& true"
5136 [(const_int 0)]
5137 {
5138 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5139
5140 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5141 : gen_lowpart (SImode, operands[0]));
5142
5143 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5144
5145 rtx and_op = gen_rtx_AND (SImode, tmp,
5146 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5147
5148 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5149
5150 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5151 DONE;
5152 }
5153 [(set_attr "length" "8")]
5154 )
5155
5156 ;; Logical left shift using SISD or Integer instruction
5157 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5158 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5159 (ashift:GPI
5160 (match_operand:GPI 1 "register_operand" "r,r,w,w")
5161 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5162 ""
5163 "@
5164 lsl\t%<w>0, %<w>1, %2
5165 lsl\t%<w>0, %<w>1, %<w>2
5166 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5167 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5168 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5169 (set_attr "arch" "*,*,simd,simd")]
5170 )
5171
5172 ;; Logical right shift using SISD or Integer instruction
5173 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5174 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5175 (lshiftrt:GPI
5176 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5177 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5178 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5179 ""
5180 "@
5181 lsr\t%<w>0, %<w>1, %2
5182 lsr\t%<w>0, %<w>1, %<w>2
5183 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5184 #
5185 #"
5186 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5187 (set_attr "arch" "*,*,simd,simd,simd")]
5188 )
5189
5190 (define_split
5191 [(set (match_operand:DI 0 "aarch64_simd_register")
5192 (lshiftrt:DI
5193 (match_operand:DI 1 "aarch64_simd_register")
5194 (match_operand:QI 2 "aarch64_simd_register")))]
5195 "TARGET_SIMD && reload_completed"
5196 [(set (match_dup 3)
5197 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5198 (set (match_dup 0)
5199 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5200 {
5201 operands[3] = gen_lowpart (QImode, operands[0]);
5202 }
5203 )
5204
5205 (define_split
5206 [(set (match_operand:SI 0 "aarch64_simd_register")
5207 (lshiftrt:SI
5208 (match_operand:SI 1 "aarch64_simd_register")
5209 (match_operand:QI 2 "aarch64_simd_register")))]
5210 "TARGET_SIMD && reload_completed"
5211 [(set (match_dup 3)
5212 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5213 (set (match_dup 0)
5214 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5215 {
5216 operands[3] = gen_lowpart (QImode, operands[0]);
5217 }
5218 )
5219
5220 ;; Arithmetic right shift using SISD or Integer instruction
5221 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5222 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5223 (ashiftrt:GPI
5224 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5225 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5226 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5227 ""
5228 "@
5229 asr\t%<w>0, %<w>1, %2
5230 asr\t%<w>0, %<w>1, %<w>2
5231 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5232 #
5233 #"
5234 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5235 (set_attr "arch" "*,*,simd,simd,simd")]
5236 )
5237
5238 (define_split
5239 [(set (match_operand:DI 0 "aarch64_simd_register")
5240 (ashiftrt:DI
5241 (match_operand:DI 1 "aarch64_simd_register")
5242 (match_operand:QI 2 "aarch64_simd_register")))]
5243 "TARGET_SIMD && reload_completed"
5244 [(set (match_dup 3)
5245 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5246 (set (match_dup 0)
5247 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5248 {
5249 operands[3] = gen_lowpart (QImode, operands[0]);
5250 }
5251 )
5252
5253 (define_split
5254 [(set (match_operand:SI 0 "aarch64_simd_register")
5255 (ashiftrt:SI
5256 (match_operand:SI 1 "aarch64_simd_register")
5257 (match_operand:QI 2 "aarch64_simd_register")))]
5258 "TARGET_SIMD && reload_completed"
5259 [(set (match_dup 3)
5260 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5261 (set (match_dup 0)
5262 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5263 {
5264 operands[3] = gen_lowpart (QImode, operands[0]);
5265 }
5266 )
5267
5268 (define_insn "*aarch64_sisd_ushl"
5269 [(set (match_operand:DI 0 "register_operand" "=w")
5270 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5271 (match_operand:QI 2 "register_operand" "w")]
5272 UNSPEC_SISD_USHL))]
5273 "TARGET_SIMD"
5274 "ushl\t%d0, %d1, %d2"
5275 [(set_attr "type" "neon_shift_reg")]
5276 )
5277
5278 (define_insn "*aarch64_ushl_2s"
5279 [(set (match_operand:SI 0 "register_operand" "=w")
5280 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5281 (match_operand:QI 2 "register_operand" "w")]
5282 UNSPEC_USHL_2S))]
5283 "TARGET_SIMD"
5284 "ushl\t%0.2s, %1.2s, %2.2s"
5285 [(set_attr "type" "neon_shift_reg")]
5286 )
5287
5288 (define_insn "*aarch64_sisd_sshl"
5289 [(set (match_operand:DI 0 "register_operand" "=w")
5290 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5291 (match_operand:QI 2 "register_operand" "w")]
5292 UNSPEC_SISD_SSHL))]
5293 "TARGET_SIMD"
5294 "sshl\t%d0, %d1, %d2"
5295 [(set_attr "type" "neon_shift_reg")]
5296 )
5297
5298 (define_insn "*aarch64_sshl_2s"
5299 [(set (match_operand:SI 0 "register_operand" "=w")
5300 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5301 (match_operand:QI 2 "register_operand" "w")]
5302 UNSPEC_SSHL_2S))]
5303 "TARGET_SIMD"
5304 "sshl\t%0.2s, %1.2s, %2.2s"
5305 [(set_attr "type" "neon_shift_reg")]
5306 )
5307
5308 (define_insn "*aarch64_sisd_neg_qi"
5309 [(set (match_operand:QI 0 "register_operand" "=w")
5310 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5311 UNSPEC_SISD_NEG))]
5312 "TARGET_SIMD"
5313 "neg\t%d0, %d1"
5314 [(set_attr "type" "neon_neg")]
5315 )
5316
5317 ;; Rotate right
5318 (define_insn "*ror<mode>3_insn"
5319 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5320 (rotatert:GPI
5321 (match_operand:GPI 1 "register_operand" "r,r")
5322 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5323 ""
5324 "@
5325 ror\\t%<w>0, %<w>1, %2
5326 ror\\t%<w>0, %<w>1, %<w>2"
5327 [(set_attr "type" "rotate_imm,shift_reg")]
5328 )
5329
5330 ;; zero_extend version of above
5331 (define_insn "*<optab>si3_insn_uxtw"
5332 [(set (match_operand:DI 0 "register_operand" "=r,r")
5333 (zero_extend:DI (SHIFT:SI
5334 (match_operand:SI 1 "register_operand" "r,r")
5335 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5336 ""
5337 "@
5338 <shift>\\t%w0, %w1, %2
5339 <shift>\\t%w0, %w1, %w2"
5340 [(set_attr "type" "bfx,shift_reg")]
5341 )
5342
5343 (define_insn "*<optab><mode>3_insn"
5344 [(set (match_operand:SHORT 0 "register_operand" "=r")
5345 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5346 (match_operand 2 "const_int_operand" "n")))]
5347 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5348 {
5349 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5350 return "<bfshift>\t%w0, %w1, %2, %3";
5351 }
5352 [(set_attr "type" "bfx")]
5353 )
5354
5355 (define_insn "*extr<mode>5_insn"
5356 [(set (match_operand:GPI 0 "register_operand" "=r")
5357 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5358 (match_operand 3 "const_int_operand" "n"))
5359 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5360 (match_operand 4 "const_int_operand" "n"))))]
5361 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5362 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5363 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5364 [(set_attr "type" "rotate_imm")]
5365 )
5366
5367 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5368 ;; so we have to match both orderings.
5369 (define_insn "*extr<mode>5_insn_alt"
5370 [(set (match_operand:GPI 0 "register_operand" "=r")
5371 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5372 (match_operand 4 "const_int_operand" "n"))
5373 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5374 (match_operand 3 "const_int_operand" "n"))))]
5375 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5376 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5377 == GET_MODE_BITSIZE (<MODE>mode))"
5378 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5379 [(set_attr "type" "rotate_imm")]
5380 )
5381
5382 ;; zero_extend version of the above
5383 (define_insn "*extrsi5_insn_uxtw"
5384 [(set (match_operand:DI 0 "register_operand" "=r")
5385 (zero_extend:DI
5386 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5387 (match_operand 3 "const_int_operand" "n"))
5388 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5389 (match_operand 4 "const_int_operand" "n")))))]
5390 "UINTVAL (operands[3]) < 32 &&
5391 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5392 "extr\\t%w0, %w1, %w2, %4"
5393 [(set_attr "type" "rotate_imm")]
5394 )
5395
5396 (define_insn "*extrsi5_insn_uxtw_alt"
5397 [(set (match_operand:DI 0 "register_operand" "=r")
5398 (zero_extend:DI
5399 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5400 (match_operand 4 "const_int_operand" "n"))
5401 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5402 (match_operand 3 "const_int_operand" "n")))))]
5403 "UINTVAL (operands[3]) < 32 &&
5404 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5405 "extr\\t%w0, %w1, %w2, %4"
5406 [(set_attr "type" "rotate_imm")]
5407 )
5408
5409 (define_insn "*ror<mode>3_insn"
5410 [(set (match_operand:GPI 0 "register_operand" "=r")
5411 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5412 (match_operand 2 "const_int_operand" "n")))]
5413 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5414 {
5415 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5416 return "ror\\t%<w>0, %<w>1, %3";
5417 }
5418 [(set_attr "type" "rotate_imm")]
5419 )
5420
5421 ;; zero_extend version of the above
5422 (define_insn "*rorsi3_insn_uxtw"
5423 [(set (match_operand:DI 0 "register_operand" "=r")
5424 (zero_extend:DI
5425 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5426 (match_operand 2 "const_int_operand" "n"))))]
5427 "UINTVAL (operands[2]) < 32"
5428 {
5429 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5430 return "ror\\t%w0, %w1, %3";
5431 }
5432 [(set_attr "type" "rotate_imm")]
5433 )
5434
5435 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5436 [(set (match_operand:GPI 0 "register_operand" "=r")
5437 (ANY_EXTEND:GPI
5438 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5439 (match_operand 2 "const_int_operand" "n"))))]
5440 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5441 {
5442 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5443 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5444 }
5445 [(set_attr "type" "bfx")]
5446 )
5447
5448 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5449 [(set (match_operand:GPI 0 "register_operand" "=r")
5450 (zero_extend:GPI
5451 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5452 (match_operand 2 "const_int_operand" "n"))))]
5453 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5454 {
5455 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5456 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5457 }
5458 [(set_attr "type" "bfx")]
5459 )
5460
5461 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5462 [(set (match_operand:GPI 0 "register_operand" "=r")
5463 (sign_extend:GPI
5464 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5465 (match_operand 2 "const_int_operand" "n"))))]
5466 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5467 {
5468 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5469 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5470 }
5471 [(set_attr "type" "bfx")]
5472 )
5473
5474 ;; -------------------------------------------------------------------
5475 ;; Bitfields
5476 ;; -------------------------------------------------------------------
5477
5478 (define_expand "<optab>"
5479 [(set (match_operand:DI 0 "register_operand")
5480 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5481 (match_operand 2
5482 "aarch64_simd_shift_imm_offset_di")
5483 (match_operand 3 "aarch64_simd_shift_imm_di")))]
5484 ""
5485 {
5486 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5487 1, GET_MODE_BITSIZE (DImode) - 1))
5488 FAIL;
5489 }
5490 )
5491
5492
5493 (define_insn "*<optab><mode>"
5494 [(set (match_operand:GPI 0 "register_operand" "=r")
5495 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5496 (match_operand 2
5497 "aarch64_simd_shift_imm_offset_<mode>" "n")
5498 (match_operand 3
5499 "aarch64_simd_shift_imm_<mode>" "n")))]
5500 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5501 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5502 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5503 [(set_attr "type" "bfx")]
5504 )
5505
5506 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5507 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5508 (define_split
5509 [(set (match_operand:DI 0 "register_operand")
5510 (zero_extract:DI (match_operand:DI 1 "register_operand")
5511 (match_operand 2
5512 "aarch64_simd_shift_imm_offset_di")
5513 (match_operand 3
5514 "aarch64_simd_shift_imm_di")))]
5515 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5516 GET_MODE_BITSIZE (DImode) - 1)
5517 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5518 == GET_MODE_BITSIZE (SImode)"
5519 [(set (match_dup 0)
5520 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5521 {
5522 operands[4] = gen_lowpart (SImode, operands[1]);
5523 }
5524 )
5525
5526 ;; Bitfield Insert (insv)
5527 (define_expand "insv<mode>"
5528 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5529 (match_operand 1 "const_int_operand")
5530 (match_operand 2 "const_int_operand"))
5531 (match_operand:GPI 3 "general_operand"))]
5532 ""
5533 {
5534 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5535 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5536 rtx value = operands[3];
5537
5538 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5539 FAIL;
5540
5541 if (CONST_INT_P (value))
5542 {
5543 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5544
5545 /* Prefer AND/OR for inserting all zeros or all ones. */
5546 if ((UINTVAL (value) & mask) == 0
5547 || (UINTVAL (value) & mask) == mask)
5548 FAIL;
5549
5550 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5551 if (width == 16 && (pos % 16) == 0)
5552 DONE;
5553 }
5554 operands[3] = force_reg (<MODE>mode, value);
5555 })
5556
5557 (define_insn "*insv_reg<mode>"
5558 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5559 (match_operand 1 "const_int_operand" "n")
5560 (match_operand 2 "const_int_operand" "n"))
5561 (match_operand:GPI 3 "register_operand" "r"))]
5562 "!(UINTVAL (operands[1]) == 0
5563 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5564 > GET_MODE_BITSIZE (<MODE>mode)))"
5565 "bfi\\t%<w>0, %<w>3, %2, %1"
5566 [(set_attr "type" "bfm")]
5567 )
5568
5569 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5570 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5571 (match_operand 1 "const_int_operand" "n")
5572 (match_operand 2 "const_int_operand" "n"))
5573 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5574 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5575 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5576 [(set_attr "type" "bfm")]
5577 )
5578
5579 ;; Match a bfi instruction where the shift of OP3 means that we are
5580 ;; actually copying the least significant bits of OP3 into OP0 by way
5581 ;; of the AND masks and the IOR instruction. A similar instruction
5582 ;; with the two parts of the IOR swapped around was never triggered
5583 ;; in a bootstrap build and test of GCC so it was not included.
5584
5585 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5586 [(set (match_operand:GPI 0 "register_operand" "=r")
5587 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5588 (match_operand:GPI 2 "const_int_operand" "n"))
5589 (and:GPI (ashift:GPI
5590 (match_operand:GPI 3 "register_operand" "r")
5591 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5592 (match_operand:GPI 5 "const_int_operand" "n"))))]
5593 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5594 UINTVAL (operands[4]),
5595 UINTVAL(operands[5]))"
5596 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5597 [(set_attr "type" "bfm")]
5598 )
5599
5600 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5601 [(set (match_operand:GPI 0 "register_operand" "=r")
5602 (ior:GPI (and:GPI (ashift:GPI
5603 (match_operand:GPI 1 "register_operand" "r")
5604 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5605 (match_operand:GPI 3 "const_int_operand" "n"))
5606 (and:GPI (match_operand:GPI 4 "register_operand" "0")
5607 (match_operand:GPI 5 "const_int_operand" "n"))))]
5608 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5609 UINTVAL (operands[2]),
5610 UINTVAL(operands[3]))"
5611 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5612 [(set_attr "type" "bfm")]
5613 )
5614
5615 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5616 ;; the shift is large enough to remove the need for an AND instruction.
5617
5618 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5619 [(set (match_operand:GPI 0 "register_operand" "=r")
5620 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5621 (match_operand:GPI 2 "const_int_operand" "n"))
5622 (ashift:GPI
5623 (match_operand:GPI 3 "register_operand" "r")
5624 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5625 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5626 UINTVAL (operands[4]),
5627 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5628 {
5629 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5630 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5631 }
5632 [(set_attr "type" "bfm")]
5633 )
5634
5635 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5636 [(set (match_operand:GPI 0 "register_operand" "=r")
5637 (ior:GPI (ashift:GPI
5638 (match_operand:GPI 1 "register_operand" "r")
5639 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5640 (and:GPI (match_operand:GPI 3 "register_operand" "0")
5641 (match_operand:GPI 4 "const_int_operand" "n"))))]
5642 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5643 UINTVAL (operands[2]),
5644 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5645 {
5646 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5647 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5648 }
5649 [(set_attr "type" "bfm")]
5650 )
5651
5652 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5653 ;; copying the least significant bits of OP3 to OP0. We need two versions
5654 ;; of the instruction to handle different checks on the constant values.
5655
5656 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5657 [(set (match_operand:GPI 0 "register_operand" "=r")
5658 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5659 (match_operand:GPI 2 "const_int_operand" "n"))
5660 (and:GPI (match_operand:GPI 3 "register_operand" "r")
5661 (match_operand:GPI 4 "const_int_operand" "n"))))]
5662 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5663 UINTVAL (operands[4]))"
5664 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5665 [(set_attr "type" "bfm")]
5666 )
5667
5668 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5669 [(set (match_operand:GPI 0 "register_operand" "=r")
5670 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5671 (match_operand:GPI 4 "const_int_operand" "n"))
5672 (and:GPI (match_operand:GPI 1 "register_operand" "0")
5673 (match_operand:GPI 2 "const_int_operand" "n"))))]
5674 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5675 UINTVAL (operands[4]))"
5676 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5677 [(set_attr "type" "bfm")]
5678 )
5679
5680 (define_insn "*extr_insv_lower_reg<mode>"
5681 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5682 (match_operand 1 "const_int_operand" "n")
5683 (const_int 0))
5684 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5685 (match_dup 1)
5686 (match_operand 3 "const_int_operand" "n")))]
5687 "!(UINTVAL (operands[1]) == 0
5688 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5689 > GET_MODE_BITSIZE (<MODE>mode)))"
5690 "bfxil\\t%<w>0, %<w>2, %3, %1"
5691 [(set_attr "type" "bfm")]
5692 )
5693
5694 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5695 [(set (match_operand:GPI 0 "register_operand" "=r")
5696 (ashift:GPI (ANY_EXTEND:GPI
5697 (match_operand:ALLX 1 "register_operand" "r"))
5698 (match_operand 2 "const_int_operand" "n")))]
5699 "UINTVAL (operands[2]) < <GPI:sizen>"
5700 {
5701 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5702 ? GEN_INT (<ALLX:sizen>)
5703 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5704 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5705 }
5706 [(set_attr "type" "bfx")]
5707 )
5708
5709 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5710
5711 (define_insn "*andim_ashift<mode>_bfiz"
5712 [(set (match_operand:GPI 0 "register_operand" "=r")
5713 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5714 (match_operand 2 "const_int_operand" "n"))
5715 (match_operand 3 "const_int_operand" "n")))]
5716 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5717 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5718 [(set_attr "type" "bfx")]
5719 )
5720
5721 ;; Match sbfiz pattern in a shift left + shift right operation.
5722
5723 (define_insn "*ashift<mode>_extv_bfiz"
5724 [(set (match_operand:GPI 0 "register_operand" "=r")
5725 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5726 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5727 (const_int 0))
5728 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5729 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5730 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5731 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5732 [(set_attr "type" "bfx")]
5733 )
5734
5735 ;; When the bit position and width of the equivalent extraction add up to 32
5736 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5737 ;; zero-extension of the X-reg.
5738 (define_split
5739 [(set (match_operand:DI 0 "register_operand")
5740 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5741 (match_operand 2 "const_int_operand"))
5742 (match_operand 3 "const_int_operand")))]
5743 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5744 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5745 == GET_MODE_BITSIZE (SImode)"
5746 [(set (match_dup 0)
5747 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5748 {
5749 operands[4] = gen_lowpart (SImode, operands[1]);
5750 }
5751 )
5752
5753 (define_insn "bswap<mode>2"
5754 [(set (match_operand:GPI 0 "register_operand" "=r")
5755 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5756 ""
5757 "rev\\t%<w>0, %<w>1"
5758 [(set_attr "type" "rev")]
5759 )
5760
5761 (define_insn "bswaphi2"
5762 [(set (match_operand:HI 0 "register_operand" "=r")
5763 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5764 ""
5765 "rev16\\t%w0, %w1"
5766 [(set_attr "type" "rev")]
5767 )
5768
5769 (define_insn "*aarch64_bfxil<mode>"
5770 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5771 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5772 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5773 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5774 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5775 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5776 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5777 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5778 {
5779 switch (which_alternative)
5780 {
5781 case 0:
5782 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5783 return "bfxil\\t%<w>0, %<w>1, 0, %3";
5784 case 1:
5785 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5786 return "bfxil\\t%<w>0, %<w>2, 0, %3";
5787 default:
5788 gcc_unreachable ();
5789 }
5790 }
5791 [(set_attr "type" "bfm")]
5792 )
5793
5794 ; Zero-extended version of above (aarch64_bfxil)
5795 (define_insn "*aarch64_bfxilsi_uxtw"
5796 [(set (match_operand:DI 0 "register_operand" "=r,r")
5797 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5798 "r,0")
5799 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5800 (and:SI (match_operand:SI 2 "register_operand" "0,r")
5801 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5802 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5803 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5804 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5805 {
5806 switch (which_alternative)
5807 {
5808 case 0:
5809 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5810 return "bfxil\\t%0, %1, 0, %3";
5811 case 1:
5812 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5813 return "bfxil\\t%0, %2, 0, %3";
5814 default:
5815 gcc_unreachable ();
5816 }
5817 }
5818 [(set_attr "type" "bfm")]
5819 )
5820
5821 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5822 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5823 ;; each valid permutation.
5824
5825 (define_insn "rev16<mode>2"
5826 [(set (match_operand:GPI 0 "register_operand" "=r")
5827 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5828 (const_int 8))
5829 (match_operand:GPI 3 "const_int_operand" "n"))
5830 (and:GPI (lshiftrt:GPI (match_dup 1)
5831 (const_int 8))
5832 (match_operand:GPI 2 "const_int_operand" "n"))))]
5833 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5834 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5835 "rev16\\t%<w>0, %<w>1"
5836 [(set_attr "type" "rev")]
5837 )
5838
5839 (define_insn "rev16<mode>2_alt"
5840 [(set (match_operand:GPI 0 "register_operand" "=r")
5841 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5842 (const_int 8))
5843 (match_operand:GPI 2 "const_int_operand" "n"))
5844 (and:GPI (ashift:GPI (match_dup 1)
5845 (const_int 8))
5846 (match_operand:GPI 3 "const_int_operand" "n"))))]
5847 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5848 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5849 "rev16\\t%<w>0, %<w>1"
5850 [(set_attr "type" "rev")]
5851 )
5852
5853 ;; zero_extend version of above
5854 (define_insn "*bswapsi2_uxtw"
5855 [(set (match_operand:DI 0 "register_operand" "=r")
5856 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5857 ""
5858 "rev\\t%w0, %w1"
5859 [(set_attr "type" "rev")]
5860 )
5861
5862 ;; -------------------------------------------------------------------
5863 ;; Floating-point intrinsics
5864 ;; -------------------------------------------------------------------
5865
5866 ;; frint floating-point round to integral standard patterns.
5867 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5868
5869 (define_insn "<frint_pattern><mode>2"
5870 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5871 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5872 FRINT))]
5873 "TARGET_FLOAT"
5874 "frint<frint_suffix>\\t%<s>0, %<s>1"
5875 [(set_attr "type" "f_rint<stype>")]
5876 )
5877
5878 ;; frcvt floating-point round to integer and convert standard patterns.
5879 ;; Expands to lbtrunc, lceil, lfloor, lround.
5880 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5881 [(set (match_operand:GPI 0 "register_operand" "=r")
5882 (FIXUORS:GPI
5883 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5884 FCVT)))]
5885 "TARGET_FLOAT"
5886 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5887 [(set_attr "type" "f_cvtf2i")]
5888 )
5889
5890 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5891 [(set (match_operand:GPI 0 "register_operand" "=r")
5892 (FIXUORS:GPI
5893 (mult:GPF
5894 (match_operand:GPF 1 "register_operand" "w")
5895 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5896 "TARGET_FLOAT
5897 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5898 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5899 {
5900 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5901 char buf[64];
5902 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5903 output_asm_insn (buf, operands);
5904 return "";
5905 }
5906 [(set_attr "type" "f_cvtf2i")]
5907 )
5908
5909 ;; fma - expand fma into patterns with the accumulator operand first since
5910 ;; reusing the accumulator results in better register allocation.
5911 ;; The register allocator considers copy preferences in operand order,
5912 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5913
5914 (define_expand "fma<mode>4"
5915 [(set (match_operand:GPF_F16 0 "register_operand")
5916 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5917 (match_operand:GPF_F16 2 "register_operand")
5918 (match_operand:GPF_F16 3 "register_operand")))]
5919 "TARGET_FLOAT"
5920 )
5921
5922 (define_insn "*aarch64_fma<mode>4"
5923 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5924 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5925 (match_operand:GPF_F16 3 "register_operand" "w")
5926 (match_operand:GPF_F16 1 "register_operand" "w")))]
5927 "TARGET_FLOAT"
5928 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5929 [(set_attr "type" "fmac<stype>")]
5930 )
5931
5932 (define_expand "fnma<mode>4"
5933 [(set (match_operand:GPF_F16 0 "register_operand")
5934 (fma:GPF_F16
5935 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5936 (match_operand:GPF_F16 2 "register_operand")
5937 (match_operand:GPF_F16 3 "register_operand")))]
5938 "TARGET_FLOAT"
5939 )
5940
5941 (define_insn "*aarch64_fnma<mode>4"
5942 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5943 (fma:GPF_F16
5944 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5945 (match_operand:GPF_F16 3 "register_operand" "w")
5946 (match_operand:GPF_F16 1 "register_operand" "w")))]
5947 "TARGET_FLOAT"
5948 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5949 [(set_attr "type" "fmac<stype>")]
5950 )
5951
5952
5953 (define_expand "fms<mode>4"
5954 [(set (match_operand:GPF 0 "register_operand")
5955 (fma:GPF (match_operand:GPF 1 "register_operand")
5956 (match_operand:GPF 2 "register_operand")
5957 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5958 "TARGET_FLOAT"
5959 )
5960
5961 (define_insn "*aarch64_fms<mode>4"
5962 [(set (match_operand:GPF 0 "register_operand" "=w")
5963 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5964 (match_operand:GPF 3 "register_operand" "w")
5965 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5966 "TARGET_FLOAT"
5967 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5968 [(set_attr "type" "fmac<s>")]
5969 )
5970
5971 (define_expand "fnms<mode>4"
5972 [(set (match_operand:GPF 0 "register_operand")
5973 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5974 (match_operand:GPF 2 "register_operand")
5975 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5976 "TARGET_FLOAT"
5977 )
5978
5979 (define_insn "*aarch64_fnms<mode>4"
5980 [(set (match_operand:GPF 0 "register_operand" "=w")
5981 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5982 (match_operand:GPF 3 "register_operand" "w")
5983 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5984 "TARGET_FLOAT"
5985 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5986 [(set_attr "type" "fmac<s>")]
5987 )
5988
5989 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5990 (define_insn "*aarch64_fnmadd<mode>4"
5991 [(set (match_operand:GPF 0 "register_operand" "=w")
5992 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5993 (match_operand:GPF 3 "register_operand" "w")
5994 (match_operand:GPF 1 "register_operand" "w"))))]
5995 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5996 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5997 [(set_attr "type" "fmac<s>")]
5998 )
5999
6000 ;; -------------------------------------------------------------------
6001 ;; Floating-point conversions
6002 ;; -------------------------------------------------------------------
6003
6004 (define_insn "extendsfdf2"
6005 [(set (match_operand:DF 0 "register_operand" "=w")
6006 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
6007 "TARGET_FLOAT"
6008 "fcvt\\t%d0, %s1"
6009 [(set_attr "type" "f_cvt")]
6010 )
6011
6012 (define_insn "extendhfsf2"
6013 [(set (match_operand:SF 0 "register_operand" "=w")
6014 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
6015 "TARGET_FLOAT"
6016 "fcvt\\t%s0, %h1"
6017 [(set_attr "type" "f_cvt")]
6018 )
6019
6020 (define_insn "extendhfdf2"
6021 [(set (match_operand:DF 0 "register_operand" "=w")
6022 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
6023 "TARGET_FLOAT"
6024 "fcvt\\t%d0, %h1"
6025 [(set_attr "type" "f_cvt")]
6026 )
6027
6028 (define_insn "truncdfsf2"
6029 [(set (match_operand:SF 0 "register_operand" "=w")
6030 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
6031 "TARGET_FLOAT"
6032 "fcvt\\t%s0, %d1"
6033 [(set_attr "type" "f_cvt")]
6034 )
6035
6036 (define_insn "truncsfhf2"
6037 [(set (match_operand:HF 0 "register_operand" "=w")
6038 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6039 "TARGET_FLOAT"
6040 "fcvt\\t%h0, %s1"
6041 [(set_attr "type" "f_cvt")]
6042 )
6043
6044 (define_insn "truncdfhf2"
6045 [(set (match_operand:HF 0 "register_operand" "=w")
6046 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6047 "TARGET_FLOAT"
6048 "fcvt\\t%h0, %d1"
6049 [(set_attr "type" "f_cvt")]
6050 )
6051
6052 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6053 ;; and making r = w more expensive
6054
6055 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
6056 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
6057 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6058 "TARGET_FLOAT"
6059 "@
6060 fcvtz<su>\t%<s>0, %<s>1
6061 fcvtz<su>\t%<w>0, %<s>1"
6062 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
6063 )
6064
6065 ;; Convert HF -> SI or DI
6066
6067 (define_insn "<optab>_trunchf<GPI:mode>2"
6068 [(set (match_operand:GPI 0 "register_operand" "=r")
6069 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6070 "TARGET_FP_F16INST"
6071 "fcvtz<su>\t%<w>0, %h1"
6072 [(set_attr "type" "f_cvtf2i")]
6073 )
6074
6075 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6076 ;; input in a fp register and output in a integer register
6077
6078 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6079 [(set (match_operand:GPI 0 "register_operand" "=r")
6080 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6081 "TARGET_FLOAT"
6082 "fcvtz<su>\t%<w>0, %<fpw>1"
6083 [(set_attr "type" "f_cvtf2i")]
6084 )
6085
6086 (define_insn "*fix_to_zero_extend<mode>di2"
6087 [(set (match_operand:DI 0 "register_operand" "=r")
6088 (zero_extend:DI
6089 (unsigned_fix:SI
6090 (match_operand:GPF 1 "register_operand" "w"))))]
6091 "TARGET_FLOAT"
6092 "fcvtzu\t%w0, %<s>1"
6093 [(set_attr "type" "f_cvtf2i")]
6094 )
6095
6096 ;; Equal width integer to fp and multiply combine.
6097 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6098 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6099 (mult:GPF (FLOATUORS:GPF
6100 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6101 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6102 "TARGET_FLOAT"
6103 {
6104 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6105 switch (which_alternative)
6106 {
6107 case 0:
6108 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6109 case 1:
6110 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6111 default:
6112 gcc_unreachable ();
6113 }
6114 }
6115 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6116 (set_attr "arch" "simd,fp")]
6117 )
6118
6119 ;; Unequal width integer to fp and multiply combine.
6120 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6121 [(set (match_operand:GPF 0 "register_operand" "=w")
6122 (mult:GPF (FLOATUORS:GPF
6123 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6124 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6125 "TARGET_FLOAT"
6126 {
6127 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6128 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6129 }
6130 [(set_attr "type" "f_cvti2f")]
6131 )
6132
6133 ;; Equal width integer to fp conversion.
6134 (define_insn "<optab><fcvt_target><GPF:mode>2"
6135 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6136 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6137 "TARGET_FLOAT"
6138 "@
6139 <su_optab>cvtf\t%<GPF:s>0, %<s>1
6140 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6141 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6142 (set_attr "arch" "simd,fp")]
6143 )
6144
6145 ;; Unequal width integer to fp conversions.
6146 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6147 [(set (match_operand:GPF 0 "register_operand" "=w")
6148 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6149 "TARGET_FLOAT"
6150 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6151 [(set_attr "type" "f_cvti2f")]
6152 )
6153
6154 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6155 ;; midend will arrange for an SImode conversion to HFmode to first go
6156 ;; through DFmode, then to HFmode. But first it will try converting
6157 ;; to DImode then down, which would match our DImode pattern below and
6158 ;; give very poor code-generation. So, we must provide our own emulation
6159 ;; of the mid-end logic.
6160
6161 (define_insn "aarch64_fp16_<optab><mode>hf2"
6162 [(set (match_operand:HF 0 "register_operand" "=w")
6163 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6164 "TARGET_FP_F16INST"
6165 "<su_optab>cvtf\t%h0, %<w>1"
6166 [(set_attr "type" "f_cvti2f")]
6167 )
6168
6169 (define_expand "<optab>sihf2"
6170 [(set (match_operand:HF 0 "register_operand")
6171 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6172 "TARGET_FLOAT"
6173 {
6174 if (TARGET_FP_F16INST)
6175 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6176 else
6177 {
6178 rtx convert_target = gen_reg_rtx (DFmode);
6179 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6180 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6181 }
6182 DONE;
6183 }
6184 )
6185
6186 ;; For DImode there is no wide enough floating-point mode that we
6187 ;; can convert through natively (TFmode would work, but requires a library
6188 ;; call). However, we know that any value >= 65504 will be rounded
6189 ;; to infinity on conversion. This is well within the range of SImode, so
6190 ;; we can:
6191 ;; Saturate to SImode.
6192 ;; Convert from that to DFmode
6193 ;; Convert from that to HFmode (phew!).
6194 ;; Note that the saturation to SImode requires the SIMD extensions. If
6195 ;; we ever need to provide this pattern where the SIMD extensions are not
6196 ;; available, we would need a different approach.
6197
6198 (define_expand "<optab>dihf2"
6199 [(set (match_operand:HF 0 "register_operand")
6200 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6201 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6202 {
6203 if (TARGET_FP_F16INST)
6204 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6205 else
6206 {
6207 rtx sat_target = gen_reg_rtx (SImode);
6208 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6209 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6210 }
6211
6212 DONE;
6213 }
6214 )
6215
6216 ;; Convert between fixed-point and floating-point (scalar modes)
6217
6218 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6219 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6220 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6221 (match_operand:SI 2 "immediate_operand" "i, i")]
6222 FCVT_F2FIXED))]
6223 ""
6224 "@
6225 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6226 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6227 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6228 (set_attr "arch" "fp,simd")]
6229 )
6230
6231 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6232 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6233 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6234 (match_operand:SI 2 "immediate_operand" "i, i")]
6235 FCVT_FIXED2F))]
6236 ""
6237 "@
6238 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6239 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6240 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6241 (set_attr "arch" "fp,simd")]
6242 )
6243
6244 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6245 [(set (match_operand:GPI 0 "register_operand" "=r")
6246 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6247 (match_operand:SI 2 "immediate_operand" "i")]
6248 FCVT_F2FIXED))]
6249 "TARGET_FP_F16INST"
6250 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6251 [(set_attr "type" "f_cvtf2i")]
6252 )
6253
6254 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6255 [(set (match_operand:HF 0 "register_operand" "=w")
6256 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6257 (match_operand:SI 2 "immediate_operand" "i")]
6258 FCVT_FIXED2F))]
6259 "TARGET_FP_F16INST"
6260 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6261 [(set_attr "type" "f_cvti2f")]
6262 )
6263
6264 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6265 [(set (match_operand:HI 0 "register_operand" "=w")
6266 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6267 (match_operand:SI 2 "immediate_operand" "i")]
6268 FCVT_F2FIXED))]
6269 "TARGET_SIMD"
6270 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6271 [(set_attr "type" "neon_fp_to_int_s")]
6272 )
6273
6274 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6275 [(set (match_operand:HF 0 "register_operand" "=w")
6276 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6277 (match_operand:SI 2 "immediate_operand" "i")]
6278 FCVT_FIXED2F))]
6279 "TARGET_SIMD"
6280 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6281 [(set_attr "type" "neon_int_to_fp_s")]
6282 )
6283
6284 ;; -------------------------------------------------------------------
6285 ;; Floating-point arithmetic
6286 ;; -------------------------------------------------------------------
6287
6288 (define_insn "add<mode>3"
6289 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6290 (plus:GPF_F16
6291 (match_operand:GPF_F16 1 "register_operand" "w")
6292 (match_operand:GPF_F16 2 "register_operand" "w")))]
6293 "TARGET_FLOAT"
6294 "fadd\\t%<s>0, %<s>1, %<s>2"
6295 [(set_attr "type" "fadd<stype>")]
6296 )
6297
6298 (define_insn "sub<mode>3"
6299 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6300 (minus:GPF_F16
6301 (match_operand:GPF_F16 1 "register_operand" "w")
6302 (match_operand:GPF_F16 2 "register_operand" "w")))]
6303 "TARGET_FLOAT"
6304 "fsub\\t%<s>0, %<s>1, %<s>2"
6305 [(set_attr "type" "fadd<stype>")]
6306 )
6307
6308 (define_insn "mul<mode>3"
6309 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6310 (mult:GPF_F16
6311 (match_operand:GPF_F16 1 "register_operand" "w")
6312 (match_operand:GPF_F16 2 "register_operand" "w")))]
6313 "TARGET_FLOAT"
6314 "fmul\\t%<s>0, %<s>1, %<s>2"
6315 [(set_attr "type" "fmul<stype>")]
6316 )
6317
6318 (define_insn "*fnmul<mode>3"
6319 [(set (match_operand:GPF 0 "register_operand" "=w")
6320 (mult:GPF
6321 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6322 (match_operand:GPF 2 "register_operand" "w")))]
6323 "TARGET_FLOAT && !flag_rounding_math"
6324 "fnmul\\t%<s>0, %<s>1, %<s>2"
6325 [(set_attr "type" "fmul<s>")]
6326 )
6327
6328 (define_insn "*fnmul<mode>3"
6329 [(set (match_operand:GPF 0 "register_operand" "=w")
6330 (neg:GPF (mult:GPF
6331 (match_operand:GPF 1 "register_operand" "w")
6332 (match_operand:GPF 2 "register_operand" "w"))))]
6333 "TARGET_FLOAT"
6334 "fnmul\\t%<s>0, %<s>1, %<s>2"
6335 [(set_attr "type" "fmul<s>")]
6336 )
6337
6338 (define_expand "div<mode>3"
6339 [(set (match_operand:GPF_F16 0 "register_operand")
6340 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6341 (match_operand:GPF_F16 2 "register_operand")))]
6342 "TARGET_FLOAT"
6343 {
6344 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6345 DONE;
6346
6347 operands[1] = force_reg (<MODE>mode, operands[1]);
6348 })
6349
6350 (define_insn "*div<mode>3"
6351 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6352 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6353 (match_operand:GPF_F16 2 "register_operand" "w")))]
6354 "TARGET_FLOAT"
6355 "fdiv\\t%<s>0, %<s>1, %<s>2"
6356 [(set_attr "type" "fdiv<stype>")]
6357 )
6358
6359 (define_insn "neg<mode>2"
6360 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6361 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6362 "TARGET_FLOAT"
6363 "fneg\\t%<s>0, %<s>1"
6364 [(set_attr "type" "ffarith<stype>")]
6365 )
6366
6367 (define_expand "sqrt<mode>2"
6368 [(set (match_operand:GPF_F16 0 "register_operand")
6369 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6370 "TARGET_FLOAT"
6371 {
6372 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6373 DONE;
6374 })
6375
6376 (define_insn "*sqrt<mode>2"
6377 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6378 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6379 "TARGET_FLOAT"
6380 "fsqrt\\t%<s>0, %<s>1"
6381 [(set_attr "type" "fsqrt<stype>")]
6382 )
6383
6384 (define_insn "abs<mode>2"
6385 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6386 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6387 "TARGET_FLOAT"
6388 "fabs\\t%<s>0, %<s>1"
6389 [(set_attr "type" "ffarith<stype>")]
6390 )
6391
6392 ;; Given that smax/smin do not specify the result when either input is NaN,
6393 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6394 ;; for smin.
6395
6396 (define_insn "smax<mode>3"
6397 [(set (match_operand:GPF 0 "register_operand" "=w")
6398 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6399 (match_operand:GPF 2 "register_operand" "w")))]
6400 "TARGET_FLOAT"
6401 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6402 [(set_attr "type" "f_minmax<s>")]
6403 )
6404
6405 (define_insn "smin<mode>3"
6406 [(set (match_operand:GPF 0 "register_operand" "=w")
6407 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6408 (match_operand:GPF 2 "register_operand" "w")))]
6409 "TARGET_FLOAT"
6410 "fminnm\\t%<s>0, %<s>1, %<s>2"
6411 [(set_attr "type" "f_minmax<s>")]
6412 )
6413
6414 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6415 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6416 ;; which implement the IEEE fmax ()/fmin () functions.
6417 (define_insn "<maxmin_uns><mode>3"
6418 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6419 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6420 (match_operand:GPF_F16 2 "register_operand" "w")]
6421 FMAXMIN_UNS))]
6422 "TARGET_FLOAT"
6423 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6424 [(set_attr "type" "f_minmax<stype>")]
6425 )
6426
6427 (define_expand "lrint<GPF:mode><GPI:mode>2"
6428 [(match_operand:GPI 0 "register_operand")
6429 (match_operand:GPF 1 "register_operand")]
6430 "TARGET_FLOAT
6431 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
6432 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6433 {
6434 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6435 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6436 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6437 DONE;
6438 }
6439 )
6440
6441 ;; For copysign (x, y), we want to generate:
6442 ;;
6443 ;; LDR d2, #(1 << 63)
6444 ;; BSL v2.8b, [y], [x]
6445 ;;
6446 ;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because
6447 ;; we expect these operations to nearly always operate on
6448 ;; floating-point values, we do not want the operation to be
6449 ;; simplified into a bit-field insert operation that operates on the
6450 ;; integer side, since typically that would involve three inter-bank
6451 ;; register copies. As we do not expect copysign to be followed by
6452 ;; other logical operations on the result, it seems preferable to keep
6453 ;; this as an unspec operation, rather than exposing the underlying
6454 ;; logic to the compiler.
6455
6456 (define_expand "copysign<GPF:mode>3"
6457 [(match_operand:GPF 0 "register_operand")
6458 (match_operand:GPF 1 "register_operand")
6459 (match_operand:GPF 2 "register_operand")]
6460 "TARGET_FLOAT && TARGET_SIMD"
6461 {
6462 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6463 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6464 << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6465 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6466 bitmask));
6467 DONE;
6468 }
6469 )
6470
6471 (define_insn "copysign<GPF:mode>3_insn"
6472 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6473 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6474 (match_operand:GPF 2 "register_operand" "w,w,0,0")
6475 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6476 UNSPEC_COPYSIGN))]
6477 "TARGET_FLOAT && TARGET_SIMD"
6478 "@
6479 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6480 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6481 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6482 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6483 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6484 )
6485
6486
6487 ;; For xorsign (x, y), we want to generate:
6488 ;;
6489 ;; LDR d2, #1<<63
6490 ;; AND v3.8B, v1.8B, v2.8B
6491 ;; EOR v0.8B, v0.8B, v3.8B
6492 ;;
6493
6494 (define_expand "xorsign<mode>3"
6495 [(match_operand:GPF 0 "register_operand")
6496 (match_operand:GPF 1 "register_operand")
6497 (match_operand:GPF 2 "register_operand")]
6498 "TARGET_FLOAT && TARGET_SIMD"
6499 {
6500
6501 machine_mode imode = <V_INT_EQUIV>mode;
6502 rtx mask = gen_reg_rtx (imode);
6503 rtx op1x = gen_reg_rtx (imode);
6504 rtx op2x = gen_reg_rtx (imode);
6505
6506 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6507 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6508 imode)));
6509
6510 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6511 lowpart_subreg (imode, operands[2],
6512 <MODE>mode)));
6513 emit_insn (gen_xor<v_int_equiv>3 (op1x,
6514 lowpart_subreg (imode, operands[1],
6515 <MODE>mode),
6516 op2x));
6517 emit_move_insn (operands[0],
6518 lowpart_subreg (<MODE>mode, op1x, imode));
6519 DONE;
6520 }
6521 )
6522
6523 ;; -------------------------------------------------------------------
6524 ;; Reload support
6525 ;; -------------------------------------------------------------------
6526 ;; Reload Scalar Floating point modes from constant pool.
6527 ;; The AArch64 port doesn't have __int128 constant move support.
6528 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6529 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6530 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6531 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6532 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6533 "TARGET_FLOAT"
6534 {
6535 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6536 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6537 DONE;
6538 }
6539 )
6540
6541 ;; Reload Vector modes from constant pool.
6542 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6543 [(set (match_operand:VALL 0 "register_operand" "=w")
6544 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6545 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6546 "TARGET_FLOAT"
6547 {
6548 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6549 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6550 DONE;
6551 }
6552 )
6553
6554 (define_expand "@aarch64_reload_mov<mode>"
6555 [(set (match_operand:TX 0 "register_operand" "=w")
6556 (match_operand:TX 1 "register_operand" "w"))
6557 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6558 ]
6559 "TARGET_FLOAT"
6560 {
6561 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6562 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6563 gen_aarch64_movtilow_tilow (op0, op1);
6564 gen_aarch64_movdi_tihigh (operands[2], op1);
6565 gen_aarch64_movtihigh_di (op0, operands[2]);
6566 DONE;
6567 }
6568 )
6569
6570 ;; The following secondary reload helpers patterns are invoked
6571 ;; after or during reload as we don't want these patterns to start
6572 ;; kicking in during the combiner.
6573
6574 (define_insn "@aarch64_movdi_<mode>low"
6575 [(set (match_operand:DI 0 "register_operand" "=r")
6576 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6577 (const_int 64) (const_int 0)))]
6578 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6579 "fmov\\t%x0, %d1"
6580 [(set_attr "type" "f_mrc")
6581 (set_attr "length" "4")
6582 ])
6583
6584 (define_insn "@aarch64_movdi_<mode>high"
6585 [(set (match_operand:DI 0 "register_operand" "=r")
6586 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6587 (const_int 64) (const_int 64)))]
6588 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6589 "fmov\\t%x0, %1.d[1]"
6590 [(set_attr "type" "f_mrc")
6591 (set_attr "length" "4")
6592 ])
6593
6594 (define_insn "@aarch64_mov<mode>high_di"
6595 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6596 (const_int 64) (const_int 64))
6597 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6598 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6599 "fmov\\t%0.d[1], %x1"
6600 [(set_attr "type" "f_mcr")
6601 (set_attr "length" "4")
6602 ])
6603
6604 (define_insn "@aarch64_mov<mode>low_di"
6605 [(set (match_operand:TX 0 "register_operand" "=w")
6606 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6607 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6608 "fmov\\t%d0, %x1"
6609 [(set_attr "type" "f_mcr")
6610 (set_attr "length" "4")
6611 ])
6612
6613 (define_insn "aarch64_movtilow_tilow"
6614 [(set (match_operand:TI 0 "register_operand" "=w")
6615 (zero_extend:TI
6616 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6617 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6618 "fmov\\t%d0, %d1"
6619 [(set_attr "type" "fmov")
6620 (set_attr "length" "4")
6621 ])
6622
6623 ;; There is a deliberate reason why the parameters of high and lo_sum's
6624 ;; don't have modes for ADRP and ADD instructions. This is to allow high
6625 ;; and lo_sum's to be used with the labels defining the jump tables in
6626 ;; rodata section.
6627
6628 (define_expand "add_losym"
6629 [(set (match_operand 0 "register_operand")
6630 (lo_sum (match_operand 1 "register_operand")
6631 (match_operand 2 "aarch64_valid_symref")))]
6632 ""
6633 {
6634 machine_mode mode = GET_MODE (operands[0]);
6635
6636 emit_insn ((mode == DImode
6637 ? gen_add_losym_di
6638 : gen_add_losym_si) (operands[0],
6639 operands[1],
6640 operands[2]));
6641 DONE;
6642 })
6643
6644 (define_insn "add_losym_<mode>"
6645 [(set (match_operand:P 0 "register_operand" "=r")
6646 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6647 (match_operand 2 "aarch64_valid_symref" "S")))]
6648 ""
6649 "add\\t%<w>0, %<w>1, :lo12:%c2"
6650 [(set_attr "type" "alu_imm")]
6651 )
6652
6653 (define_insn "ldr_got_small_<mode>"
6654 [(set (match_operand:PTR 0 "register_operand" "=r")
6655 (unspec:PTR [(mem:PTR (lo_sum:PTR
6656 (match_operand:PTR 1 "register_operand" "r")
6657 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6658 UNSPEC_GOTSMALLPIC))]
6659 ""
6660 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6661 [(set_attr "type" "load_<ldst_sz>")]
6662 )
6663
6664 (define_insn "ldr_got_small_sidi"
6665 [(set (match_operand:DI 0 "register_operand" "=r")
6666 (zero_extend:DI
6667 (unspec:SI [(mem:SI (lo_sum:DI
6668 (match_operand:DI 1 "register_operand" "r")
6669 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6670 UNSPEC_GOTSMALLPIC)))]
6671 "TARGET_ILP32"
6672 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6673 [(set_attr "type" "load_4")]
6674 )
6675
6676 (define_insn "ldr_got_small_28k_<mode>"
6677 [(set (match_operand:PTR 0 "register_operand" "=r")
6678 (unspec:PTR [(mem:PTR (lo_sum:PTR
6679 (match_operand:PTR 1 "register_operand" "r")
6680 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6681 UNSPEC_GOTSMALLPIC28K))]
6682 ""
6683 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6684 [(set_attr "type" "load_<ldst_sz>")]
6685 )
6686
6687 (define_insn "ldr_got_small_28k_sidi"
6688 [(set (match_operand:DI 0 "register_operand" "=r")
6689 (zero_extend:DI
6690 (unspec:SI [(mem:SI (lo_sum:DI
6691 (match_operand:DI 1 "register_operand" "r")
6692 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6693 UNSPEC_GOTSMALLPIC28K)))]
6694 "TARGET_ILP32"
6695 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6696 [(set_attr "type" "load_4")]
6697 )
6698
6699 (define_insn "ldr_got_tiny"
6700 [(set (match_operand:DI 0 "register_operand" "=r")
6701 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6702 UNSPEC_GOTTINYPIC))]
6703 ""
6704 "ldr\\t%0, %L1"
6705 [(set_attr "type" "load_8")]
6706 )
6707
6708 (define_insn "aarch64_load_tp_hard"
6709 [(set (match_operand:DI 0 "register_operand" "=r")
6710 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6711 ""
6712 "mrs\\t%0, tpidr_el0"
6713 [(set_attr "type" "mrs")]
6714 )
6715
6716 ;; The TLS ABI specifically requires that the compiler does not schedule
6717 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6718 ;; Therefore we treat the stubs as an atomic sequence.
6719 (define_expand "tlsgd_small_<mode>"
6720 [(parallel [(set (match_operand 0 "register_operand")
6721 (call (mem:DI (match_dup 2)) (const_int 1)))
6722 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6723 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6724 (clobber (reg:DI LR_REGNUM))])]
6725 ""
6726 {
6727 operands[2] = aarch64_tls_get_addr ();
6728 })
6729
6730 (define_insn "*tlsgd_small_<mode>"
6731 [(set (match_operand 0 "register_operand" "")
6732 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6733 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6734 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6735 (clobber (reg:DI LR_REGNUM))
6736 ]
6737 ""
6738 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6739 [(set_attr "type" "call")
6740 (set_attr "length" "16")])
6741
6742 (define_insn "tlsie_small_<mode>"
6743 [(set (match_operand:PTR 0 "register_operand" "=r")
6744 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6745 UNSPEC_GOTSMALLTLS))]
6746 ""
6747 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6748 [(set_attr "type" "load_4")
6749 (set_attr "length" "8")]
6750 )
6751
6752 (define_insn "tlsie_small_sidi"
6753 [(set (match_operand:DI 0 "register_operand" "=r")
6754 (zero_extend:DI
6755 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6756 UNSPEC_GOTSMALLTLS)))]
6757 ""
6758 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6759 [(set_attr "type" "load_4")
6760 (set_attr "length" "8")]
6761 )
6762
6763 (define_insn "tlsie_tiny_<mode>"
6764 [(set (match_operand:PTR 0 "register_operand" "=&r")
6765 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6766 (match_operand:PTR 2 "register_operand" "r")]
6767 UNSPEC_GOTTINYTLS))]
6768 ""
6769 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6770 [(set_attr "type" "multiple")
6771 (set_attr "length" "8")]
6772 )
6773
6774 (define_insn "tlsie_tiny_sidi"
6775 [(set (match_operand:DI 0 "register_operand" "=&r")
6776 (zero_extend:DI
6777 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6778 (match_operand:DI 2 "register_operand" "r")
6779 ]
6780 UNSPEC_GOTTINYTLS)))]
6781 ""
6782 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6783 [(set_attr "type" "multiple")
6784 (set_attr "length" "8")]
6785 )
6786
6787 (define_insn "tlsle12_<mode>"
6788 [(set (match_operand:P 0 "register_operand" "=r")
6789 (unspec:P [(match_operand:P 1 "register_operand" "r")
6790 (match_operand 2 "aarch64_tls_le_symref" "S")]
6791 UNSPEC_TLSLE12))]
6792 ""
6793 "add\\t%<w>0, %<w>1, #%L2";
6794 [(set_attr "type" "alu_sreg")
6795 (set_attr "length" "4")]
6796 )
6797
6798 (define_insn "tlsle24_<mode>"
6799 [(set (match_operand:P 0 "register_operand" "=r")
6800 (unspec:P [(match_operand:P 1 "register_operand" "r")
6801 (match_operand 2 "aarch64_tls_le_symref" "S")]
6802 UNSPEC_TLSLE24))]
6803 ""
6804 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6805 [(set_attr "type" "multiple")
6806 (set_attr "length" "8")]
6807 )
6808
6809 (define_insn "tlsle32_<mode>"
6810 [(set (match_operand:P 0 "register_operand" "=r")
6811 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6812 UNSPEC_TLSLE32))]
6813 ""
6814 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6815 [(set_attr "type" "multiple")
6816 (set_attr "length" "8")]
6817 )
6818
6819 (define_insn "tlsle48_<mode>"
6820 [(set (match_operand:P 0 "register_operand" "=r")
6821 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6822 UNSPEC_TLSLE48))]
6823 ""
6824 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6825 [(set_attr "type" "multiple")
6826 (set_attr "length" "12")]
6827 )
6828
6829 (define_expand "tlsdesc_small_<mode>"
6830 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6831 "TARGET_TLS_DESC"
6832 {
6833 if (TARGET_SVE)
6834 {
6835 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode);
6836 rtx_insn *call
6837 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
6838 RTL_CONST_CALL_P (call) = 1;
6839 }
6840 else
6841 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6842 DONE;
6843 }
6844 )
6845
6846 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6847 ;; R0 and LR.
6848 (define_insn "tlsdesc_small_advsimd_<mode>"
6849 [(set (reg:PTR R0_REGNUM)
6850 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6851 UNSPEC_TLSDESC))
6852 (clobber (reg:DI LR_REGNUM))
6853 (clobber (reg:CC CC_REGNUM))
6854 (clobber (match_scratch:DI 1 "=r"))
6855 (use (reg:DI FP_REGNUM))]
6856 "TARGET_TLS_DESC && !TARGET_SVE"
6857 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6858 [(set_attr "type" "call")
6859 (set_attr "length" "16")])
6860
6861 ;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
6862 ;; describing the extra call-preserved guarantees. This would work
6863 ;; for non-SVE too, but avoiding a call is probably better if we can.
6864 (define_insn "tlsdesc_small_sve_<mode>"
6865 [(set (reg:PTR R0_REGNUM)
6866 (call (mem:DI (unspec:PTR
6867 [(match_operand 0 "aarch64_valid_symref")]
6868 UNSPEC_TLSDESC))
6869 (const_int 0)))
6870 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
6871 (clobber (reg:DI LR_REGNUM))
6872 (clobber (match_scratch:DI 2 "=r"))]
6873 "TARGET_TLS_DESC && TARGET_SVE"
6874 "adrp\\tx0, %A0\;ldr\\t%<w>2, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%2"
6875 [(set_attr "type" "call")
6876 (set_attr "length" "16")])
6877
6878 (define_insn "stack_tie"
6879 [(set (mem:BLK (scratch))
6880 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6881 (match_operand:DI 1 "register_operand" "rk")]
6882 UNSPEC_PRLG_STK))]
6883 ""
6884 ""
6885 [(set_attr "length" "0")]
6886 )
6887
6888 (define_insn "aarch64_fjcvtzs"
6889 [(set (match_operand:SI 0 "register_operand" "=r")
6890 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
6891 UNSPEC_FJCVTZS))]
6892 "TARGET_JSCVT"
6893 "fjcvtzs\\t%w0, %d1"
6894 [(set_attr "type" "f_cvtf2i")]
6895 )
6896
6897 ;; Pointer authentication patterns are always provided. In architecture
6898 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6899 ;; This lets the user write portable software which authenticates pointers
6900 ;; when run on something which implements ARMv8.3-A, and which runs
6901 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6902 ;; implemented.
6903
6904 ;; Signing/Authenticating R30 using SP as the salt.
6905
6906 (define_insn "<pauth_mnem_prefix>sp"
6907 [(set (reg:DI R30_REGNUM)
6908 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6909 ""
6910 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6911 )
6912
6913 ;; Signing/Authenticating X17 using X16 as the salt.
6914
6915 (define_insn "<pauth_mnem_prefix>1716"
6916 [(set (reg:DI R17_REGNUM)
6917 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6918 ""
6919 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6920 )
6921
6922 ;; Stripping the signature in R30.
6923
6924 (define_insn "xpaclri"
6925 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6926 ""
6927 "hint\t7 // xpaclri"
6928 )
6929
6930 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6931 ;; all of memory. This blocks insns from being moved across this point.
6932
6933 (define_insn "blockage"
6934 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6935 ""
6936 ""
6937 [(set_attr "length" "0")
6938 (set_attr "type" "block")]
6939 )
6940
6941 (define_insn "probe_stack_range"
6942 [(set (match_operand:DI 0 "register_operand" "=rk")
6943 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6944 (match_operand:DI 2 "register_operand" "r")]
6945 UNSPECV_PROBE_STACK_RANGE))]
6946 ""
6947 {
6948 return aarch64_output_probe_stack_range (operands[0], operands[2]);
6949 }
6950 [(set_attr "length" "32")]
6951 )
6952
6953 ;; This instruction is used to generate the stack clash stack adjustment and
6954 ;; probing loop. We can't change the control flow during prologue and epilogue
6955 ;; code generation. So we must emit a volatile unspec and expand it later on.
6956
6957 (define_insn "@probe_sve_stack_clash_<mode>"
6958 [(set (match_operand:P 0 "register_operand" "=rk")
6959 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6960 (match_operand:P 2 "register_operand" "r")
6961 (match_operand:P 3 "const_int_operand" "n")
6962 (match_operand:P 4 "aarch64_plus_immediate" "L")]
6963 UNSPECV_PROBE_STACK_RANGE))]
6964 "TARGET_SVE"
6965 {
6966 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6967 operands[3], operands[4]);
6968 }
6969 [(set_attr "length" "28")]
6970 )
6971
6972 ;; Named pattern for expanding thread pointer reference.
6973 (define_expand "get_thread_pointerdi"
6974 [(match_operand:DI 0 "register_operand")]
6975 ""
6976 {
6977 rtx tmp = aarch64_load_tp (operands[0]);
6978 if (tmp != operands[0])
6979 emit_move_insn (operands[0], tmp);
6980 DONE;
6981 })
6982
6983 ;; Named patterns for stack smashing protection.
6984 (define_expand "stack_protect_set"
6985 [(match_operand 0 "memory_operand")
6986 (match_operand 1 "memory_operand")]
6987 ""
6988 {
6989 machine_mode mode = GET_MODE (operands[0]);
6990 if (aarch64_stack_protector_guard != SSP_GLOBAL)
6991 {
6992 /* Generate access through the system register. */
6993 rtx tmp_reg = gen_reg_rtx (mode);
6994 if (mode == DImode)
6995 {
6996 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6997 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6998 GEN_INT (aarch64_stack_protector_guard_offset)));
6999 }
7000 else
7001 {
7002 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7003 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7004 GEN_INT (aarch64_stack_protector_guard_offset)));
7005
7006 }
7007 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7008 }
7009
7010 emit_insn ((mode == DImode
7011 ? gen_stack_protect_set_di
7012 : gen_stack_protect_set_si) (operands[0], operands[1]));
7013 DONE;
7014 })
7015
7016 (define_insn "reg_stack_protect_address_<mode>"
7017 [(set (match_operand:PTR 0 "register_operand" "=r")
7018 (unspec:PTR [(const_int 0)]
7019 UNSPEC_SSP_SYSREG))]
7020 "aarch64_stack_protector_guard != SSP_GLOBAL"
7021 {
7022 char buf[150];
7023 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7024 aarch64_stack_protector_guard_reg_str);
7025 output_asm_insn (buf, operands);
7026 return "";
7027 }
7028 [(set_attr "type" "mrs")])
7029
7030 ;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
7031 ;; canary value does not live beyond the life of this sequence.
7032 (define_insn "stack_protect_set_<mode>"
7033 [(set (match_operand:PTR 0 "memory_operand" "=m")
7034 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7035 UNSPEC_SP_SET))
7036 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7037 ""
7038 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
7039 [(set_attr "length" "12")
7040 (set_attr "type" "multiple")])
7041
7042 (define_expand "stack_protect_test"
7043 [(match_operand 0 "memory_operand")
7044 (match_operand 1 "memory_operand")
7045 (match_operand 2)]
7046 ""
7047 {
7048 rtx result;
7049 machine_mode mode = GET_MODE (operands[0]);
7050
7051 result = gen_reg_rtx(mode);
7052 if (aarch64_stack_protector_guard != SSP_GLOBAL)
7053 {
7054 /* Generate access through the system register. The
7055 sequence we want here is the access
7056 of the stack offset to come with
7057 mrs scratch_reg, <system_register>
7058 add scratch_reg, scratch_reg, :lo12:offset. */
7059 rtx tmp_reg = gen_reg_rtx (mode);
7060 if (mode == DImode)
7061 {
7062 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7063 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7064 GEN_INT (aarch64_stack_protector_guard_offset)));
7065 }
7066 else
7067 {
7068 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7069 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7070 GEN_INT (aarch64_stack_protector_guard_offset)));
7071
7072 }
7073 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7074 }
7075 emit_insn ((mode == DImode
7076 ? gen_stack_protect_test_di
7077 : gen_stack_protect_test_si) (result,
7078 operands[0],
7079 operands[1]));
7080
7081 if (mode == DImode)
7082 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7083 result, const0_rtx, operands[2]));
7084 else
7085 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7086 result, const0_rtx, operands[2]));
7087 DONE;
7088 })
7089
7090 (define_insn "stack_protect_test_<mode>"
7091 [(set (match_operand:PTR 0 "register_operand" "=r")
7092 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7093 (match_operand:PTR 2 "memory_operand" "m")]
7094 UNSPEC_SP_TEST))
7095 (clobber (match_scratch:PTR 3 "=&r"))]
7096 ""
7097 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7098 [(set_attr "length" "12")
7099 (set_attr "type" "multiple")])
7100
7101 ;; Write Floating-point Control Register.
7102 (define_insn "set_fpcr"
7103 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7104 ""
7105 "msr\\tfpcr, %0"
7106 [(set_attr "type" "mrs")])
7107
7108 ;; Read Floating-point Control Register.
7109 (define_insn "get_fpcr"
7110 [(set (match_operand:SI 0 "register_operand" "=r")
7111 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7112 ""
7113 "mrs\\t%0, fpcr"
7114 [(set_attr "type" "mrs")])
7115
7116 ;; Write Floating-point Status Register.
7117 (define_insn "set_fpsr"
7118 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7119 ""
7120 "msr\\tfpsr, %0"
7121 [(set_attr "type" "mrs")])
7122
7123 ;; Read Floating-point Status Register.
7124 (define_insn "get_fpsr"
7125 [(set (match_operand:SI 0 "register_operand" "=r")
7126 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7127 ""
7128 "mrs\\t%0, fpsr"
7129 [(set_attr "type" "mrs")])
7130
7131
7132 ;; Define the subtract-one-and-jump insns so loop.c
7133 ;; knows what to generate.
7134 (define_expand "doloop_end"
7135 [(use (match_operand 0 "" "")) ; loop pseudo
7136 (use (match_operand 1 "" ""))] ; label
7137 "optimize > 0 && flag_modulo_sched"
7138 {
7139 rtx s0;
7140 rtx bcomp;
7141 rtx loc_ref;
7142 rtx cc_reg;
7143 rtx insn;
7144 rtx cmp;
7145
7146 /* Currently SMS relies on the do-loop pattern to recognize loops
7147 where (1) the control part consists of all insns defining and/or
7148 using a certain 'count' register and (2) the loop count can be
7149 adjusted by modifying this register prior to the loop.
7150 ??? The possible introduction of a new block to initialize the
7151 new IV can potentially affect branch optimizations. */
7152
7153 if (GET_MODE (operands[0]) != DImode)
7154 FAIL;
7155
7156 s0 = operands [0];
7157 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7158
7159 cmp = XVECEXP (PATTERN (insn), 0, 0);
7160 cc_reg = SET_DEST (cmp);
7161 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7162 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7163 emit_jump_insn (gen_rtx_SET (pc_rtx,
7164 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7165 loc_ref, pc_rtx)));
7166 DONE;
7167 })
7168
7169 ;; Track speculation through conditional branches. We assume that
7170 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7171 (define_insn "speculation_tracker"
7172 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7173 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7174 UNSPEC_SPECULATION_TRACKER))]
7175 ""
7176 {
7177 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7178 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7179 return "";
7180 }
7181 [(set_attr "type" "csel")]
7182 )
7183
7184 ;; BTI <target> instructions
7185 (define_insn "bti_noarg"
7186 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7187 ""
7188 "hint\t32 // bti"
7189 [(set_attr "type" "no_insn")]
7190 )
7191
7192 (define_insn "bti_c"
7193 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7194 ""
7195 "hint\t34 // bti c"
7196 [(set_attr "type" "no_insn")]
7197 )
7198
7199 (define_insn "bti_j"
7200 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7201 ""
7202 "hint\t36 // bti j"
7203 [(set_attr "type" "no_insn")]
7204 )
7205
7206 (define_insn "bti_jc"
7207 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7208 ""
7209 "hint\t38 // bti jc"
7210 [(set_attr "type" "no_insn")]
7211 )
7212
7213 ;; Hard speculation barrier.
7214 (define_insn "speculation_barrier"
7215 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7216 ""
7217 "isb\;dsb\\tsy"
7218 [(set_attr "length" "8")
7219 (set_attr "type" "block")
7220 (set_attr "speculation_barrier" "true")]
7221 )
7222
7223 ;; Support for __builtin_speculation_safe_value when we have speculation
7224 ;; tracking enabled. Use the speculation tracker to decide whether to
7225 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7226 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7227 [(set (match_operand:ALLI_TI 0 "register_operand")
7228 (unspec_volatile:ALLI_TI
7229 [(match_operand:ALLI_TI 1 "register_operand")
7230 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7231 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7232 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7233 ""
7234 "
7235 {
7236 if (operands[2] == const0_rtx)
7237 {
7238 rtx tracker;
7239 if (<MODE>mode == TImode)
7240 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7241 else
7242 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7243
7244 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7245 tracker));
7246 DONE;
7247 }
7248 }
7249 "
7250 )
7251
7252 ;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
7253 ;; encoding for CSDB, but will work in older versions of the assembler.
7254 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7255 [(set (match_operand:ALLI 0 "register_operand" "=r")
7256 (unspec_volatile:ALLI
7257 [(match_operand:ALLI 1 "register_operand" "r")
7258 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7259 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7260 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7261 ""
7262 {
7263 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7264 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7265 operands);
7266 return "";
7267 }
7268 [(set_attr "length" "12")
7269 (set_attr "type" "block")
7270 (set_attr "speculation_barrier" "true")]
7271 )
7272
7273 ;; Pattern to match despeculate_copyti
7274 (define_insn "*despeculate_copyti_insn"
7275 [(set (match_operand:TI 0 "register_operand" "=r")
7276 (unspec_volatile:TI
7277 [(match_operand:TI 1 "register_operand" "r")
7278 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7279 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7280 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7281 ""
7282 {
7283 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7284 output_asm_insn
7285 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7286 operands);
7287 return "";
7288 }
7289 [(set_attr "length" "16")
7290 (set_attr "type" "block")
7291 (set_attr "speculation_barrier" "true")]
7292 )
7293
7294 (define_insn "despeculate_simple<ALLI:mode>"
7295 [(set (match_operand:ALLI 0 "register_operand" "=r")
7296 (unspec_volatile:ALLI
7297 [(match_operand:ALLI 1 "register_operand" "r")
7298 (use (match_operand:ALLI 2 "register_operand" ""))]
7299 UNSPECV_SPECULATION_BARRIER))]
7300 ""
7301 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7302 [(set_attr "type" "block")
7303 (set_attr "length" "8")
7304 (set_attr "speculation_barrier" "true")]
7305 )
7306
7307 (define_insn "despeculate_simpleti"
7308 [(set (match_operand:TI 0 "register_operand" "=r")
7309 (unspec_volatile:TI
7310 [(match_operand:TI 1 "register_operand" "r")
7311 (use (match_operand:DI 2 "register_operand" ""))]
7312 UNSPECV_SPECULATION_BARRIER))]
7313 ""
7314 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7315 [(set_attr "type" "block")
7316 (set_attr "length" "12")
7317 (set_attr "speculation_barrier" "true")]
7318 )
7319
7320 (define_insn "aarch64_<frintnzs_op><mode>"
7321 [(set (match_operand:VSFDF 0 "register_operand" "=w")
7322 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7323 FRINTNZX))]
7324 "TARGET_FRINT && TARGET_FLOAT
7325 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7326 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7327 [(set_attr "type" "f_rint<stype>")]
7328 )
7329
7330 ;; Transactional Memory Extension (TME) instructions.
7331
7332 (define_insn "tstart"
7333 [(set (match_operand:DI 0 "register_operand" "=r")
7334 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7335 (clobber (mem:BLK (scratch)))]
7336 "TARGET_TME"
7337 "tstart\\t%0"
7338 [(set_attr "type" "tme")]
7339 )
7340
7341 (define_insn "ttest"
7342 [(set (match_operand:DI 0 "register_operand" "=r")
7343 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7344 (clobber (mem:BLK (scratch)))]
7345 "TARGET_TME"
7346 "ttest\\t%0"
7347 [(set_attr "type" "tme")]
7348 )
7349
7350 (define_insn "tcommit"
7351 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7352 (clobber (mem:BLK (scratch)))]
7353 "TARGET_TME"
7354 "tcommit"
7355 [(set_attr "type" "tme")]
7356 )
7357
7358 (define_insn "tcancel"
7359 [(unspec_volatile:BLK
7360 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7361 (clobber (mem:BLK (scratch)))]
7362 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7363 "tcancel\\t#%0"
7364 [(set_attr "type" "tme")]
7365 )
7366
7367 (define_insn "aarch64_rndr"
7368 [(set (match_operand:DI 0 "register_operand" "=r")
7369 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR))
7370 (set (reg:CC_Z CC_REGNUM)
7371 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))]
7372 "TARGET_RNG"
7373 "mrs\t%0, RNDR"
7374 [(set_attr "type" "mrs")]
7375 )
7376
7377 (define_insn "aarch64_rndrrs"
7378 [(set (match_operand:DI 0 "register_operand" "=r")
7379 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS))
7380 (set (reg:CC_Z CC_REGNUM)
7381 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))]
7382 "TARGET_RNG"
7383 "mrs\t%0, RNDRRS"
7384 [(set_attr "type" "mrs")]
7385 )
7386
7387 ;; AdvSIMD Stuff
7388 (include "aarch64-simd.md")
7389
7390 ;; Atomic Operations
7391 (include "atomics.md")
7392
7393 ;; ldp/stp peephole patterns
7394 (include "aarch64-ldpstp.md")
7395
7396 ;; SVE.
7397 (include "aarch64-sve.md")
7398
7399 ;; SVE2.
7400 (include "aarch64-sve2.md")
GNU Compiler Collection
This page took 0.344268 seconds and 6 git commands to generate.