]> gcc.gnu.org Git - gcc.git/blame - gcc/config/aarch64/aarch64-builtins.c
git://gcc.gnu.org / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
aarch64: Fix status return logic in RNG intrinsics
[gcc.git] / gcc / config / aarch64 / aarch64-builtins.c
CommitLineData
43e9d192 1/* Builtins' description for AArch64 SIMD architecture.
99dee823 2 Copyright (C) 2011-2021 Free Software Foundation, Inc.
43e9d192
IB		 3 Contributed by ARM Ltd.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
8fcc61f8
RS		 21#define IN_TARGET_CODE 1
22
43e9d192
IB		 23#include "config.h"
24#include "system.h"
25#include "coretypes.h"
26#include "tm.h"
c7131fb2 27#include "function.h"
c7131fb2 28#include "basic-block.h"
e11c4407 29#include "rtl.h"
c7131fb2
AM		 30#include "tree.h"
31#include "gimple.h"
4d0cdd0c 32#include "memmodel.h"
e11c4407
AM		 33#include "tm_p.h"
34#include "expmed.h"
35#include "optabs.h"
36#include "recog.h"
37#include "diagnostic-core.h"
40e23961 38#include "fold-const.h"
d8a2d370 39#include "stor-layout.h"
36566b39 40#include "explow.h"
43e9d192 41#include "expr.h"
43e9d192 42#include "langhooks.h"
5be5c238 43#include "gimple-iterator.h"
10766209 44#include "case-cfn-macros.h"
9d63f43b 45#include "emit-rtl.h"
31427b97
RS		 46#include "stringpool.h"
47#include "attribs.h"
43e9d192 48
0d4a1197
RS		 49#define v8qi_UP E_V8QImode
50#define v4hi_UP E_V4HImode
51#define v4hf_UP E_V4HFmode
52#define v2si_UP E_V2SImode
53#define v2sf_UP E_V2SFmode
54#define v1df_UP E_V1DFmode
55#define di_UP E_DImode
56#define df_UP E_DFmode
57#define v16qi_UP E_V16QImode
58#define v8hi_UP E_V8HImode
59#define v8hf_UP E_V8HFmode
60#define v4si_UP E_V4SImode
61#define v4sf_UP E_V4SFmode
62#define v2di_UP E_V2DImode
63#define v2df_UP E_V2DFmode
64#define ti_UP E_TImode
65#define oi_UP E_OImode
66#define ci_UP E_CImode
67#define xi_UP E_XImode
68#define si_UP E_SImode
69#define sf_UP E_SFmode
70#define hi_UP E_HImode
71#define hf_UP E_HFmode
72#define qi_UP E_QImode
abbe1ed2
SMW		 73#define bf_UP E_BFmode
74#define v4bf_UP E_V4BFmode
75#define v8bf_UP E_V8BFmode
43e9d192
IB		 76#define UP(X) X##_UP
77
b5828b4b
JG		 78#define SIMD_MAX_BUILTIN_ARGS 5
79
80enum aarch64_type_qualifiers
43e9d192 81{
b5828b4b
JG		 82 /* T foo. */
83 qualifier_none = 0x0,
84 /* unsigned T foo. */
85 qualifier_unsigned = 0x1, /* 1 << 0 */
86 /* const T foo. */
87 qualifier_const = 0x2, /* 1 << 1 */
88 /* T *foo. */
89 qualifier_pointer = 0x4, /* 1 << 2 */
b5828b4b
JG		 90 /* Used when expanding arguments if an operand could
91 be an immediate. */
92 qualifier_immediate = 0x8, /* 1 << 3 */
93 qualifier_maybe_immediate = 0x10, /* 1 << 4 */
94 /* void foo (...). */
95 qualifier_void = 0x20, /* 1 << 5 */
96 /* Some patterns may have internal operands, this qualifier is an
97 instruction to the initialisation code to skip this operand. */
98 qualifier_internal = 0x40, /* 1 << 6 */
99 /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
100 rather than using the type of the operand. */
101 qualifier_map_mode = 0x80, /* 1 << 7 */
102 /* qualifier_pointer | qualifier_map_mode */
103 qualifier_pointer_map_mode = 0x84,
e625e715 104 /* qualifier_const | qualifier_pointer | qualifier_map_mode */
6db1ec94
JG		 105 qualifier_const_pointer_map_mode = 0x86,
106 /* Polynomial types. */
2a49c16d
AL		 107 qualifier_poly = 0x100,
108 /* Lane indices - must be in range, and flipped for bigendian. */
4d0a0237
CB		 109 qualifier_lane_index = 0x200,
110 /* Lane indices for single lane structure loads and stores. */
9d63f43b
TC		 111 qualifier_struct_load_store_lane_index = 0x400,
112 /* Lane indices selected in pairs. - must be in range, and flipped for
113 bigendian. */
114 qualifier_lane_pair_index = 0x800,
8c197c85
SMW		 115 /* Lane indices selected in quadtuplets. - must be in range, and flipped for
116 bigendian. */
117 qualifier_lane_quadtup_index = 0x1000,
b5828b4b 118};
43e9d192 119
bf592b2f 120/* Flags that describe what a function might do. */
121const unsigned int FLAG_NONE = 0U;
122const unsigned int FLAG_READ_FPCR = 1U << 0;
123const unsigned int FLAG_RAISE_FP_EXCEPTIONS = 1U << 1;
124const unsigned int FLAG_READ_MEMORY = 1U << 2;
125const unsigned int FLAG_PREFETCH_MEMORY = 1U << 3;
126const unsigned int FLAG_WRITE_MEMORY = 1U << 4;
127
35ffd4d1 128/* Not all FP intrinsics raise FP exceptions or read FPCR register,
129 use this flag to suppress it. */
130const unsigned int FLAG_AUTO_FP = 1U << 5;
131
bf592b2f 132const unsigned int FLAG_FP = FLAG_READ_FPCR | FLAG_RAISE_FP_EXCEPTIONS;
133const unsigned int FLAG_ALL = FLAG_READ_FPCR | FLAG_RAISE_FP_EXCEPTIONS
134 | FLAG_READ_MEMORY | FLAG_PREFETCH_MEMORY | FLAG_WRITE_MEMORY;
2d5aad69 135const unsigned int FLAG_STORE = FLAG_WRITE_MEMORY | FLAG_AUTO_FP;
e8062ad4 136const unsigned int FLAG_LOAD = FLAG_READ_MEMORY | FLAG_AUTO_FP;
bf592b2f 137
43e9d192
IB		 138typedef struct
139{
140 const char *name;
ef4bddc2 141 machine_mode mode;
342be7f7
JG		 142 const enum insn_code code;
143 unsigned int fcode;
b5828b4b 144 enum aarch64_type_qualifiers *qualifiers;
bf592b2f 145 unsigned int flags;
43e9d192
IB		 146} aarch64_simd_builtin_datum;
147
b5828b4b
JG		 148static enum aarch64_type_qualifiers
149aarch64_types_unop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
8f905d69 150 = { qualifier_none, qualifier_none };
b5828b4b 151#define TYPES_UNOP (aarch64_types_unop_qualifiers)
5a7a4e80
TB		 152static enum aarch64_type_qualifiers
153aarch64_types_unopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
154 = { qualifier_unsigned, qualifier_unsigned };
155#define TYPES_UNOPU (aarch64_types_unopu_qualifiers)
b5828b4b 156static enum aarch64_type_qualifiers
a579f4c7
JW		 157aarch64_types_unopus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
158 = { qualifier_unsigned, qualifier_none };
159#define TYPES_UNOPUS (aarch64_types_unopus_qualifiers)
160static enum aarch64_type_qualifiers
b5828b4b
JG		 161aarch64_types_binop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
162 = { qualifier_none, qualifier_none, qualifier_maybe_immediate };
163#define TYPES_BINOP (aarch64_types_binop_qualifiers)
164static enum aarch64_type_qualifiers
5a7a4e80
TB		 165aarch64_types_binopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
166 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned };
167#define TYPES_BINOPU (aarch64_types_binopu_qualifiers)
7baa225d 168static enum aarch64_type_qualifiers
de10bcce
AL		 169aarch64_types_binop_uus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
170 = { qualifier_unsigned, qualifier_unsigned, qualifier_none };
171#define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers)
172static enum aarch64_type_qualifiers
918621d3
AL		 173aarch64_types_binop_ssu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
174 = { qualifier_none, qualifier_none, qualifier_unsigned };
175#define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers)
176static enum aarch64_type_qualifiers
daef0a8c
JW		 177aarch64_types_binop_uss_qualifiers[SIMD_MAX_BUILTIN_ARGS]
178 = { qualifier_unsigned, qualifier_none, qualifier_none };
179#define TYPES_BINOP_USS (aarch64_types_binop_uss_qualifiers)
180static enum aarch64_type_qualifiers
7baa225d
TB		 181aarch64_types_binopp_qualifiers[SIMD_MAX_BUILTIN_ARGS]
182 = { qualifier_poly, qualifier_poly, qualifier_poly };
183#define TYPES_BINOPP (aarch64_types_binopp_qualifiers)
184
5a7a4e80 185static enum aarch64_type_qualifiers
b5828b4b
JG		 186aarch64_types_ternop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
187 = { qualifier_none, qualifier_none, qualifier_none, qualifier_none };
188#define TYPES_TERNOP (aarch64_types_ternop_qualifiers)
30442682 189static enum aarch64_type_qualifiers
2a49c16d
AL		 190aarch64_types_ternop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
191 = { qualifier_none, qualifier_none, qualifier_none, qualifier_lane_index };
192#define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers)
193static enum aarch64_type_qualifiers
30442682
TB		 194aarch64_types_ternopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
195 = { qualifier_unsigned, qualifier_unsigned,
196 qualifier_unsigned, qualifier_unsigned };
197#define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers)
27086ea3 198static enum aarch64_type_qualifiers
0b839322
WD		 199aarch64_types_ternopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
200 = { qualifier_unsigned, qualifier_unsigned,
201 qualifier_unsigned, qualifier_lane_index };
202#define TYPES_TERNOPU_LANE (aarch64_types_ternopu_lane_qualifiers)
203static enum aarch64_type_qualifiers
27086ea3
MC		 204aarch64_types_ternopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
205 = { qualifier_unsigned, qualifier_unsigned,
206 qualifier_unsigned, qualifier_immediate };
207#define TYPES_TERNOPUI (aarch64_types_ternopu_imm_qualifiers)
8c197c85
SMW		 208static enum aarch64_type_qualifiers
209aarch64_types_ternop_ssus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
210 = { qualifier_none, qualifier_none, qualifier_unsigned, qualifier_none };
211#define TYPES_TERNOP_SSUS (aarch64_types_ternop_ssus_qualifiers)
27086ea3 212
30442682 213
9d63f43b
TC		 214static enum aarch64_type_qualifiers
215aarch64_types_quadop_lane_pair_qualifiers[SIMD_MAX_BUILTIN_ARGS]
216 = { qualifier_none, qualifier_none, qualifier_none,
217 qualifier_none, qualifier_lane_pair_index };
218#define TYPES_QUADOP_LANE_PAIR (aarch64_types_quadop_lane_pair_qualifiers)
b5828b4b 219static enum aarch64_type_qualifiers
2a49c16d 220aarch64_types_quadop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
b5828b4b 221 = { qualifier_none, qualifier_none, qualifier_none,
2a49c16d
AL		 222 qualifier_none, qualifier_lane_index };
223#define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
7a08d813
TC		 224static enum aarch64_type_qualifiers
225aarch64_types_quadopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
226 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
227 qualifier_unsigned, qualifier_lane_index };
228#define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers)
b5828b4b 229
8c197c85
SMW		 230static enum aarch64_type_qualifiers
231aarch64_types_quadopssus_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS]
232 = { qualifier_none, qualifier_none, qualifier_unsigned,
233 qualifier_none, qualifier_lane_quadtup_index };
234#define TYPES_QUADOPSSUS_LANE_QUADTUP \
235 (aarch64_types_quadopssus_lane_quadtup_qualifiers)
236static enum aarch64_type_qualifiers
237aarch64_types_quadopsssu_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS]
238 = { qualifier_none, qualifier_none, qualifier_none,
239 qualifier_unsigned, qualifier_lane_quadtup_index };
240#define TYPES_QUADOPSSSU_LANE_QUADTUP \
241 (aarch64_types_quadopsssu_lane_quadtup_qualifiers)
242
27086ea3
MC		 243static enum aarch64_type_qualifiers
244aarch64_types_quadopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
245 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
246 qualifier_unsigned, qualifier_immediate };
247#define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers)
248
159b8724
TC		 249static enum aarch64_type_qualifiers
250aarch64_types_binop_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
251 = { qualifier_poly, qualifier_none, qualifier_immediate };
252#define TYPES_GETREGP (aarch64_types_binop_imm_p_qualifiers)
b5828b4b 253static enum aarch64_type_qualifiers
2a49c16d 254aarch64_types_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
b5828b4b 255 = { qualifier_none, qualifier_none, qualifier_immediate };
2a49c16d
AL		 256#define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
257#define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers)
b5828b4b 258static enum aarch64_type_qualifiers
de10bcce
AL		 259aarch64_types_shift_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
260 = { qualifier_unsigned, qualifier_none, qualifier_immediate };
261#define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers)
262static enum aarch64_type_qualifiers
1f0e9e34
JG		 263aarch64_types_fcvt_from_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
264 = { qualifier_none, qualifier_unsigned, qualifier_immediate };
265#define TYPES_FCVTIMM_SUS (aarch64_types_fcvt_from_unsigned_qualifiers)
266static enum aarch64_type_qualifiers
252c7556
AV		 267aarch64_types_unsigned_shift_qualifiers[SIMD_MAX_BUILTIN_ARGS]
268 = { qualifier_unsigned, qualifier_unsigned, qualifier_immediate };
269#define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers)
05f1883c
DC		 270#define TYPES_USHIFT2IMM (aarch64_types_ternopu_imm_qualifiers)
271static enum aarch64_type_qualifiers
272aarch64_types_shift2_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
273 = { qualifier_unsigned, qualifier_unsigned, qualifier_none, qualifier_immediate };
274#define TYPES_SHIFT2IMM_UUSS (aarch64_types_shift2_to_unsigned_qualifiers)
de10bcce 275
252c7556 276static enum aarch64_type_qualifiers
159b8724
TC		 277aarch64_types_ternop_s_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
278 = { qualifier_none, qualifier_none, qualifier_poly, qualifier_immediate};
279#define TYPES_SETREGP (aarch64_types_ternop_s_imm_p_qualifiers)
280static enum aarch64_type_qualifiers
281aarch64_types_ternop_s_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
282 = { qualifier_none, qualifier_none, qualifier_none, qualifier_immediate};
283#define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers)
284#define TYPES_SHIFTINSERT (aarch64_types_ternop_s_imm_qualifiers)
285#define TYPES_SHIFTACC (aarch64_types_ternop_s_imm_qualifiers)
05f1883c 286#define TYPES_SHIFT2IMM (aarch64_types_ternop_s_imm_qualifiers)
159b8724
TC		 287
288static enum aarch64_type_qualifiers
289aarch64_types_ternop_p_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
290 = { qualifier_poly, qualifier_poly, qualifier_poly, qualifier_immediate};
291#define TYPES_SHIFTINSERTP (aarch64_types_ternop_p_imm_qualifiers)
b5828b4b 292
de10bcce
AL		 293static enum aarch64_type_qualifiers
294aarch64_types_unsigned_shiftacc_qualifiers[SIMD_MAX_BUILTIN_ARGS]
295 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
296 qualifier_immediate };
297#define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
298
299
b5828b4b
JG		 300static enum aarch64_type_qualifiers
301aarch64_types_combine_qualifiers[SIMD_MAX_BUILTIN_ARGS]
302 = { qualifier_none, qualifier_none, qualifier_none };
303#define TYPES_COMBINE (aarch64_types_combine_qualifiers)
304
159b8724
TC		 305static enum aarch64_type_qualifiers
306aarch64_types_combine_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
307 = { qualifier_poly, qualifier_poly, qualifier_poly };
308#define TYPES_COMBINEP (aarch64_types_combine_p_qualifiers)
309
b5828b4b
JG		 310static enum aarch64_type_qualifiers
311aarch64_types_load1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
312 = { qualifier_none, qualifier_const_pointer_map_mode };
313#define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
314#define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
3ec1be97
CB		 315static enum aarch64_type_qualifiers
316aarch64_types_loadstruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
317 = { qualifier_none, qualifier_const_pointer_map_mode,
4d0a0237 318 qualifier_none, qualifier_struct_load_store_lane_index };
3ec1be97 319#define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
b5828b4b 320
46e778c4
JG		 321static enum aarch64_type_qualifiers
322aarch64_types_bsl_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
323 = { qualifier_poly, qualifier_unsigned,
324 qualifier_poly, qualifier_poly };
325#define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers)
326static enum aarch64_type_qualifiers
327aarch64_types_bsl_s_qualifiers[SIMD_MAX_BUILTIN_ARGS]
328 = { qualifier_none, qualifier_unsigned,
329 qualifier_none, qualifier_none };
330#define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers)
331static enum aarch64_type_qualifiers
332aarch64_types_bsl_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
333 = { qualifier_unsigned, qualifier_unsigned,
334 qualifier_unsigned, qualifier_unsigned };
335#define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers)
336
b5828b4b
JG		 337/* The first argument (return type) of a store should be void type,
338 which we represent with qualifier_void. Their first operand will be
339 a DImode pointer to the location to store to, so we must use
340 qualifier_map_mode | qualifier_pointer to build a pointer to the
341 element type of the vector. */
342static enum aarch64_type_qualifiers
159b8724
TC		 343aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
344 = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly };
345#define TYPES_STORE1P (aarch64_types_store1_p_qualifiers)
346static enum aarch64_type_qualifiers
b5828b4b
JG		 347aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
348 = { qualifier_void, qualifier_pointer_map_mode, qualifier_none };
349#define TYPES_STORE1 (aarch64_types_store1_qualifiers)
350#define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
ba081b77
JG		 351static enum aarch64_type_qualifiers
352aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
353 = { qualifier_void, qualifier_pointer_map_mode,
4d0a0237 354 qualifier_none, qualifier_struct_load_store_lane_index };
ba081b77 355#define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
b5828b4b 356
0ddec79f
JG		 357#define CF0(N, X) CODE_FOR_aarch64_##N##X
358#define CF1(N, X) CODE_FOR_##N##X##1
359#define CF2(N, X) CODE_FOR_##N##X##2
360#define CF3(N, X) CODE_FOR_##N##X##3
361#define CF4(N, X) CODE_FOR_##N##X##4
362#define CF10(N, X) CODE_FOR_##N##X
363
bf592b2f 364#define VAR1(T, N, MAP, FLAG, A) \
365 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T, FLAG_##FLAG},
366#define VAR2(T, N, MAP, FLAG, A, B) \
367 VAR1 (T, N, MAP, FLAG, A) \
368 VAR1 (T, N, MAP, FLAG, B)
369#define VAR3(T, N, MAP, FLAG, A, B, C) \
370 VAR2 (T, N, MAP, FLAG, A, B) \
371 VAR1 (T, N, MAP, FLAG, C)
372#define VAR4(T, N, MAP, FLAG, A, B, C, D) \
373 VAR3 (T, N, MAP, FLAG, A, B, C) \
374 VAR1 (T, N, MAP, FLAG, D)
375#define VAR5(T, N, MAP, FLAG, A, B, C, D, E) \
376 VAR4 (T, N, MAP, FLAG, A, B, C, D) \
377 VAR1 (T, N, MAP, FLAG, E)
378#define VAR6(T, N, MAP, FLAG, A, B, C, D, E, F) \
379 VAR5 (T, N, MAP, FLAG, A, B, C, D, E) \
380 VAR1 (T, N, MAP, FLAG, F)
381#define VAR7(T, N, MAP, FLAG, A, B, C, D, E, F, G) \
382 VAR6 (T, N, MAP, FLAG, A, B, C, D, E, F) \
383 VAR1 (T, N, MAP, FLAG, G)
384#define VAR8(T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \
385 VAR7 (T, N, MAP, FLAG, A, B, C, D, E, F, G) \
386 VAR1 (T, N, MAP, FLAG, H)
387#define VAR9(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \
388 VAR8 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \
389 VAR1 (T, N, MAP, FLAG, I)
390#define VAR10(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \
391 VAR9 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \
392 VAR1 (T, N, MAP, FLAG, J)
393#define VAR11(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \
394 VAR10 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \
395 VAR1 (T, N, MAP, FLAG, K)
396#define VAR12(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \
397 VAR11 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \
398 VAR1 (T, N, MAP, FLAG, L)
399#define VAR13(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \
400 VAR12 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \
401 VAR1 (T, N, MAP, FLAG, M)
402#define VAR14(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
403 VAR13 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \
404 VAR1 (T, X, MAP, FLAG, N)
405#define VAR15(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
406 VAR14 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
407 VAR1 (T, X, MAP, FLAG, O)
408#define VAR16(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) \
409 VAR15 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
410 VAR1 (T, X, MAP, FLAG, P)
342be7f7 411
f421c516 412#include "aarch64-builtin-iterators.h"
43e9d192
IB		 413
414static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
342be7f7
JG		 415#include "aarch64-simd-builtins.def"
416};
417
5d357f26
KT		 418/* There's only 8 CRC32 builtins. Probably not worth their own .def file. */
419#define AARCH64_CRC32_BUILTINS \
420 CRC32_BUILTIN (crc32b, QI) \
421 CRC32_BUILTIN (crc32h, HI) \
422 CRC32_BUILTIN (crc32w, SI) \
423 CRC32_BUILTIN (crc32x, DI) \
424 CRC32_BUILTIN (crc32cb, QI) \
425 CRC32_BUILTIN (crc32ch, HI) \
426 CRC32_BUILTIN (crc32cw, SI) \
427 CRC32_BUILTIN (crc32cx, DI)
428
9d63f43b
TC		 429/* The next 8 FCMLA instrinsics require some special handling compared the
430 normal simd intrinsics. */
431#define AARCH64_SIMD_FCMLA_LANEQ_BUILTINS \
432 FCMLA_LANEQ_BUILTIN (0, v2sf, fcmla, V2SF, false) \
433 FCMLA_LANEQ_BUILTIN (90, v2sf, fcmla, V2SF, false) \
434 FCMLA_LANEQ_BUILTIN (180, v2sf, fcmla, V2SF, false) \
435 FCMLA_LANEQ_BUILTIN (270, v2sf, fcmla, V2SF, false) \
436 FCMLA_LANEQ_BUILTIN (0, v4hf, fcmla_laneq, V4HF, true) \
437 FCMLA_LANEQ_BUILTIN (90, v4hf, fcmla_laneq, V4HF, true) \
438 FCMLA_LANEQ_BUILTIN (180, v4hf, fcmla_laneq, V4HF, true) \
439 FCMLA_LANEQ_BUILTIN (270, v4hf, fcmla_laneq, V4HF, true) \
440
5d357f26
KT		 441typedef struct
442{
443 const char *name;
ef4bddc2 444 machine_mode mode;
5d357f26
KT		 445 const enum insn_code icode;
446 unsigned int fcode;
447} aarch64_crc_builtin_datum;
448
9d63f43b
TC		 449/* Hold information about how to expand the FCMLA_LANEQ builtins. */
450typedef struct
451{
452 const char *name;
453 machine_mode mode;
454 const enum insn_code icode;
455 unsigned int fcode;
456 bool lane;
457} aarch64_fcmla_laneq_builtin_datum;
458
5d357f26
KT		 459#define CRC32_BUILTIN(N, M) \
460 AARCH64_BUILTIN_##N,
461
9d63f43b
TC		 462#define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
463 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M,
464
342be7f7 465#undef VAR1
bf592b2f 466#define VAR1(T, N, MAP, FLAG, A) \
e993fea1 467 AARCH64_SIMD_BUILTIN_##T##_##N##A,
342be7f7
JG		 468
469enum aarch64_builtins
470{
471 AARCH64_BUILTIN_MIN,
aa87aced
KV		 472
473 AARCH64_BUILTIN_GET_FPCR,
474 AARCH64_BUILTIN_SET_FPCR,
475 AARCH64_BUILTIN_GET_FPSR,
476 AARCH64_BUILTIN_SET_FPSR,
477
0d7e5fa6
AC		 478 AARCH64_BUILTIN_GET_FPCR64,
479 AARCH64_BUILTIN_SET_FPCR64,
480 AARCH64_BUILTIN_GET_FPSR64,
481 AARCH64_BUILTIN_SET_FPSR64,
482
a6fc00da
BH		 483 AARCH64_BUILTIN_RSQRT_DF,
484 AARCH64_BUILTIN_RSQRT_SF,
485 AARCH64_BUILTIN_RSQRT_V2DF,
486 AARCH64_BUILTIN_RSQRT_V2SF,
487 AARCH64_BUILTIN_RSQRT_V4SF,
342be7f7 488 AARCH64_SIMD_BUILTIN_BASE,
661fce82 489 AARCH64_SIMD_BUILTIN_LANE_CHECK,
342be7f7 490#include "aarch64-simd-builtins.def"
661fce82
AL		 491 /* The first enum element which is based on an insn_data pattern. */
492 AARCH64_SIMD_PATTERN_START = AARCH64_SIMD_BUILTIN_LANE_CHECK + 1,
493 AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_PATTERN_START
494 + ARRAY_SIZE (aarch64_simd_builtin_data) - 1,
5d357f26
KT		 495 AARCH64_CRC32_BUILTIN_BASE,
496 AARCH64_CRC32_BUILTINS
497 AARCH64_CRC32_BUILTIN_MAX,
312492bd
JW		 498 /* ARMv8.3-A Pointer Authentication Builtins. */
499 AARCH64_PAUTH_BUILTIN_AUTIA1716,
500 AARCH64_PAUTH_BUILTIN_PACIA1716,
8fc16d72
ST		 501 AARCH64_PAUTH_BUILTIN_AUTIB1716,
502 AARCH64_PAUTH_BUILTIN_PACIB1716,
312492bd 503 AARCH64_PAUTH_BUILTIN_XPACLRI,
9d63f43b
TC		 504 /* Special cased Armv8.3-A Complex FMA by Lane quad Builtins. */
505 AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE,
506 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
e1d5d19e
KT		 507 /* Builtin for Arm8.3-a Javascript conversion instruction. */
508 AARCH64_JSCVT,
89626179
SD		 509 /* TME builtins. */
510 AARCH64_TME_BUILTIN_TSTART,
511 AARCH64_TME_BUILTIN_TCOMMIT,
512 AARCH64_TME_BUILTIN_TTEST,
513 AARCH64_TME_BUILTIN_TCANCEL,
c5dc215d
KT		 514 /* Armv8.5-a RNG instruction builtins. */
515 AARCH64_BUILTIN_RNG_RNDR,
516 AARCH64_BUILTIN_RNG_RNDRRS,
ef01e6bb
DZ		 517 /* MEMTAG builtins. */
518 AARCH64_MEMTAG_BUILTIN_START,
519 AARCH64_MEMTAG_BUILTIN_IRG,
520 AARCH64_MEMTAG_BUILTIN_GMI,
521 AARCH64_MEMTAG_BUILTIN_SUBP,
522 AARCH64_MEMTAG_BUILTIN_INC_TAG,
523 AARCH64_MEMTAG_BUILTIN_SET_TAG,
524 AARCH64_MEMTAG_BUILTIN_GET_TAG,
525 AARCH64_MEMTAG_BUILTIN_END,
342be7f7 526 AARCH64_BUILTIN_MAX
43e9d192
IB		 527};
528
5d357f26
KT		 529#undef CRC32_BUILTIN
530#define CRC32_BUILTIN(N, M) \
0d4a1197 531 {"__builtin_aarch64_"#N, E_##M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N},
5d357f26
KT		 532
533static aarch64_crc_builtin_datum aarch64_crc_builtin_data[] = {
534 AARCH64_CRC32_BUILTINS
535};
536
9d63f43b
TC		 537
538#undef FCMLA_LANEQ_BUILTIN
539#define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
540 {"__builtin_aarch64_fcmla_laneq"#I#N, E_##M##mode, CODE_FOR_aarch64_##X##I##N, \
541 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, T},
542
543/* This structure contains how to manage the mapping form the builtin to the
544 instruction to generate in the backend and how to invoke the instruction. */
5eb9ac1e 545static aarch64_fcmla_laneq_builtin_datum aarch64_fcmla_lane_builtin_data[] = {
9d63f43b
TC		 546 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
547};
548
5d357f26
KT		 549#undef CRC32_BUILTIN
550
119103ca
JG		 551static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX];
552
43e9d192
IB		 553#define NUM_DREG_TYPES 6
554#define NUM_QREG_TYPES 6
555
f9d53c27
TB		 556/* Internal scalar builtin types. These types are used to support
557 neon intrinsic builtins. They are _not_ user-visible types. Therefore
558 the mangling for these types are implementation defined. */
559const char *aarch64_scalar_builtin_types[] = {
560 "__builtin_aarch64_simd_qi",
561 "__builtin_aarch64_simd_hi",
562 "__builtin_aarch64_simd_si",
7c369485 563 "__builtin_aarch64_simd_hf",
f9d53c27
TB		 564 "__builtin_aarch64_simd_sf",
565 "__builtin_aarch64_simd_di",
566 "__builtin_aarch64_simd_df",
567 "__builtin_aarch64_simd_poly8",
568 "__builtin_aarch64_simd_poly16",
569 "__builtin_aarch64_simd_poly64",
570 "__builtin_aarch64_simd_poly128",
571 "__builtin_aarch64_simd_ti",
572 "__builtin_aarch64_simd_uqi",
573 "__builtin_aarch64_simd_uhi",
574 "__builtin_aarch64_simd_usi",
575 "__builtin_aarch64_simd_udi",
576 "__builtin_aarch64_simd_ei",
577 "__builtin_aarch64_simd_oi",
578 "__builtin_aarch64_simd_ci",
579 "__builtin_aarch64_simd_xi",
e603cd43 580 "__builtin_aarch64_simd_bf",
f9d53c27
TB		 581 NULL
582};
b5828b4b 583
f9d53c27
TB		 584#define ENTRY(E, M, Q, G) E,
585enum aarch64_simd_type
586{
587#include "aarch64-simd-builtin-types.def"
588 ARM_NEON_H_TYPES_LAST
589};
590#undef ENTRY
b5828b4b 591
f9d53c27 592struct aarch64_simd_type_info
b5828b4b 593{
f9d53c27
TB		 594 enum aarch64_simd_type type;
595
596 /* Internal type name. */
597 const char *name;
598
599 /* Internal type name(mangled). The mangled names conform to the
600 AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
601 Appendix A). To qualify for emission with the mangled names defined in
602 that document, a vector type must not only be of the correct mode but also
603 be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
604 types are registered by aarch64_init_simd_builtin_types (). In other
605 words, vector types defined in other ways e.g. via vector_size attribute
606 will get default mangled names. */
607 const char *mangle;
608
609 /* Internal type. */
610 tree itype;
611
612 /* Element type. */
b5828b4b
JG		 613 tree eltype;
614
f9d53c27
TB		 615 /* Machine mode the internal type maps to. */
616 enum machine_mode mode;
b5828b4b 617
f9d53c27
TB		 618 /* Qualifiers. */
619 enum aarch64_type_qualifiers q;
620};
621
622#define ENTRY(E, M, Q, G) \
0d4a1197 623 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q},
f9d53c27
TB		 624static struct aarch64_simd_type_info aarch64_simd_types [] = {
625#include "aarch64-simd-builtin-types.def"
626};
627#undef ENTRY
628
629static tree aarch64_simd_intOI_type_node = NULL_TREE;
f9d53c27
TB		 630static tree aarch64_simd_intCI_type_node = NULL_TREE;
631static tree aarch64_simd_intXI_type_node = NULL_TREE;
632
1b62ed4f
JG		 633/* The user-visible __fp16 type, and a pointer to that type. Used
634 across the back-end. */
635tree aarch64_fp16_type_node = NULL_TREE;
636tree aarch64_fp16_ptr_type_node = NULL_TREE;
637
abbe1ed2
SMW		 638/* Back-end node type for brain float (bfloat) types. */
639tree aarch64_bf16_type_node = NULL_TREE;
640tree aarch64_bf16_ptr_type_node = NULL_TREE;
641
6d4d616a 642/* Wrapper around add_builtin_function. NAME is the name of the built-in
072a8b8f 643 function, TYPE is the function type, CODE is the function subcode
644 (relative to AARCH64_BUILTIN_GENERAL), and ATTRS is the function
645 attributes. */
6d4d616a 646static tree
072a8b8f 647aarch64_general_add_builtin (const char *name, tree type, unsigned int code,
648 tree attrs = NULL_TREE)
6d4d616a
RS		 649{
650 code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_GENERAL;
651 return add_builtin_function (name, type, code, BUILT_IN_MD,
072a8b8f 652 NULL, attrs);
6d4d616a
RS		 653}
654
f9d53c27
TB		 655static const char *
656aarch64_mangle_builtin_scalar_type (const_tree type)
657{
658 int i = 0;
659
660 while (aarch64_scalar_builtin_types[i] != NULL)
b5828b4b 661 {
f9d53c27
TB		 662 const char *name = aarch64_scalar_builtin_types[i];
663
664 if (TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
665 && DECL_NAME (TYPE_NAME (type))
666 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))), name))
667 return aarch64_scalar_builtin_types[i];
668 i++;
669 }
670 return NULL;
b5828b4b
JG		 671}
672
f9d53c27
TB		 673static const char *
674aarch64_mangle_builtin_vector_type (const_tree type)
b5828b4b 675{
31427b97
RS		 676 tree attrs = TYPE_ATTRIBUTES (type);
677 if (tree attr = lookup_attribute ("Advanced SIMD type", attrs))
678 {
679 tree mangled_name = TREE_VALUE (TREE_VALUE (attr));
680 return IDENTIFIER_POINTER (mangled_name);
681 }
f9d53c27
TB		 682
683 return NULL;
6db1ec94
JG		 684}
685
f9d53c27 686const char *
6d4d616a 687aarch64_general_mangle_builtin_type (const_tree type)
6db1ec94 688{
f9d53c27
TB		 689 const char *mangle;
690 /* Walk through all the AArch64 builtins types tables to filter out the
691 incoming type. */
692 if ((mangle = aarch64_mangle_builtin_vector_type (type))
693 || (mangle = aarch64_mangle_builtin_scalar_type (type)))
694 return mangle;
695
696 return NULL;
6db1ec94
JG		 697}
698
f9d53c27 699static tree
b8506a8a 700aarch64_simd_builtin_std_type (machine_mode mode,
f9d53c27 701 enum aarch64_type_qualifiers q)
6db1ec94 702{
f9d53c27
TB		 703#define QUAL_TYPE(M) \
704 ((q == qualifier_none) ? int##M##_type_node : unsigned_int##M##_type_node);
705 switch (mode)
706 {
4e10a5a7 707 case E_QImode:
f9d53c27 708 return QUAL_TYPE (QI);
4e10a5a7 709 case E_HImode:
f9d53c27 710 return QUAL_TYPE (HI);
4e10a5a7 711 case E_SImode:
f9d53c27 712 return QUAL_TYPE (SI);
4e10a5a7 713 case E_DImode:
f9d53c27 714 return QUAL_TYPE (DI);
4e10a5a7 715 case E_TImode:
f9d53c27 716 return QUAL_TYPE (TI);
4e10a5a7 717 case E_OImode:
f9d53c27 718 return aarch64_simd_intOI_type_node;
4e10a5a7 719 case E_CImode:
f9d53c27 720 return aarch64_simd_intCI_type_node;
4e10a5a7 721 case E_XImode:
f9d53c27 722 return aarch64_simd_intXI_type_node;
4e10a5a7 723 case E_HFmode:
71a11456 724 return aarch64_fp16_type_node;
4e10a5a7 725 case E_SFmode:
f9d53c27 726 return float_type_node;
4e10a5a7 727 case E_DFmode:
f9d53c27 728 return double_type_node;
abbe1ed2
SMW		 729 case E_BFmode:
730 return aarch64_bf16_type_node;
f9d53c27
TB		 731 default:
732 gcc_unreachable ();
733 }
734#undef QUAL_TYPE
6db1ec94
JG		 735}
736
f9d53c27 737static tree
b8506a8a 738aarch64_lookup_simd_builtin_type (machine_mode mode,
f9d53c27 739 enum aarch64_type_qualifiers q)
6db1ec94 740{
f9d53c27
TB		 741 int i;
742 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
743
744 /* Non-poly scalar modes map to standard types not in the table. */
745 if (q != qualifier_poly && !VECTOR_MODE_P (mode))
746 return aarch64_simd_builtin_std_type (mode, q);
747
748 for (i = 0; i < nelts; i++)
749 if (aarch64_simd_types[i].mode == mode
750 && aarch64_simd_types[i].q == q)
751 return aarch64_simd_types[i].itype;
752
753 return NULL_TREE;
b5828b4b
JG		 754}
755
f9d53c27 756static tree
b8506a8a 757aarch64_simd_builtin_type (machine_mode mode,
f9d53c27
TB		 758 bool unsigned_p, bool poly_p)
759{
760 if (poly_p)
761 return aarch64_lookup_simd_builtin_type (mode, qualifier_poly);
762 else if (unsigned_p)
763 return aarch64_lookup_simd_builtin_type (mode, qualifier_unsigned);
764 else
765 return aarch64_lookup_simd_builtin_type (mode, qualifier_none);
766}
767
af55e82d 768static void
f9d53c27 769aarch64_init_simd_builtin_types (void)
43e9d192 770{
f9d53c27
TB		 771 int i;
772 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
773 tree tdecl;
774
775 /* Init all the element types built by the front-end. */
776 aarch64_simd_types[Int8x8_t].eltype = intQI_type_node;
777 aarch64_simd_types[Int8x16_t].eltype = intQI_type_node;
778 aarch64_simd_types[Int16x4_t].eltype = intHI_type_node;
779 aarch64_simd_types[Int16x8_t].eltype = intHI_type_node;
780 aarch64_simd_types[Int32x2_t].eltype = intSI_type_node;
781 aarch64_simd_types[Int32x4_t].eltype = intSI_type_node;
782 aarch64_simd_types[Int64x1_t].eltype = intDI_type_node;
783 aarch64_simd_types[Int64x2_t].eltype = intDI_type_node;
784 aarch64_simd_types[Uint8x8_t].eltype = unsigned_intQI_type_node;
785 aarch64_simd_types[Uint8x16_t].eltype = unsigned_intQI_type_node;
786 aarch64_simd_types[Uint16x4_t].eltype = unsigned_intHI_type_node;
787 aarch64_simd_types[Uint16x8_t].eltype = unsigned_intHI_type_node;
788 aarch64_simd_types[Uint32x2_t].eltype = unsigned_intSI_type_node;
789 aarch64_simd_types[Uint32x4_t].eltype = unsigned_intSI_type_node;
790 aarch64_simd_types[Uint64x1_t].eltype = unsigned_intDI_type_node;
791 aarch64_simd_types[Uint64x2_t].eltype = unsigned_intDI_type_node;
792
793 /* Poly types are a world of their own. */
794 aarch64_simd_types[Poly8_t].eltype = aarch64_simd_types[Poly8_t].itype =
795 build_distinct_type_copy (unsigned_intQI_type_node);
bcee52c4
MS		 796 /* Prevent front-ends from transforming Poly8_t arrays into string
797 literals. */
798 TYPE_STRING_FLAG (aarch64_simd_types[Poly8_t].eltype) = false;
799
f9d53c27
TB		 800 aarch64_simd_types[Poly16_t].eltype = aarch64_simd_types[Poly16_t].itype =
801 build_distinct_type_copy (unsigned_intHI_type_node);
802 aarch64_simd_types[Poly64_t].eltype = aarch64_simd_types[Poly64_t].itype =
803 build_distinct_type_copy (unsigned_intDI_type_node);
804 aarch64_simd_types[Poly128_t].eltype = aarch64_simd_types[Poly128_t].itype =
805 build_distinct_type_copy (unsigned_intTI_type_node);
806 /* Init poly vector element types with scalar poly types. */
807 aarch64_simd_types[Poly8x8_t].eltype = aarch64_simd_types[Poly8_t].itype;
808 aarch64_simd_types[Poly8x16_t].eltype = aarch64_simd_types[Poly8_t].itype;
809 aarch64_simd_types[Poly16x4_t].eltype = aarch64_simd_types[Poly16_t].itype;
810 aarch64_simd_types[Poly16x8_t].eltype = aarch64_simd_types[Poly16_t].itype;
811 aarch64_simd_types[Poly64x1_t].eltype = aarch64_simd_types[Poly64_t].itype;
812 aarch64_simd_types[Poly64x2_t].eltype = aarch64_simd_types[Poly64_t].itype;
813
814 /* Continue with standard types. */
71a11456
AL		 815 aarch64_simd_types[Float16x4_t].eltype = aarch64_fp16_type_node;
816 aarch64_simd_types[Float16x8_t].eltype = aarch64_fp16_type_node;
f9d53c27
TB		 817 aarch64_simd_types[Float32x2_t].eltype = float_type_node;
818 aarch64_simd_types[Float32x4_t].eltype = float_type_node;
819 aarch64_simd_types[Float64x1_t].eltype = double_type_node;
820 aarch64_simd_types[Float64x2_t].eltype = double_type_node;
821
abbe1ed2
SMW		 822 /* Init Bfloat vector types with underlying __bf16 type. */
823 aarch64_simd_types[Bfloat16x4_t].eltype = aarch64_bf16_type_node;
824 aarch64_simd_types[Bfloat16x8_t].eltype = aarch64_bf16_type_node;
825
f9d53c27
TB		 826 for (i = 0; i < nelts; i++)
827 {
828 tree eltype = aarch64_simd_types[i].eltype;
b8506a8a 829 machine_mode mode = aarch64_simd_types[i].mode;
f9d53c27
TB		 830
831 if (aarch64_simd_types[i].itype == NULL)
b96824c4 832 {
31427b97
RS		 833 tree type = build_vector_type (eltype, GET_MODE_NUNITS (mode));
834 type = build_distinct_type_copy (type);
835 SET_TYPE_STRUCTURAL_EQUALITY (type);
836
837 tree mangled_name = get_identifier (aarch64_simd_types[i].mangle);
838 tree value = tree_cons (NULL_TREE, mangled_name, NULL_TREE);
839 TYPE_ATTRIBUTES (type)
840 = tree_cons (get_identifier ("Advanced SIMD type"), value,
841 TYPE_ATTRIBUTES (type));
842 aarch64_simd_types[i].itype = type;
b96824c4 843 }
f9d53c27
TB		 844
845 tdecl = add_builtin_type (aarch64_simd_types[i].name,
846 aarch64_simd_types[i].itype);
847 TYPE_NAME (aarch64_simd_types[i].itype) = tdecl;
f9d53c27 848 }
43e9d192 849
f9d53c27
TB		 850#define AARCH64_BUILD_SIGNED_TYPE(mode) \
851 make_signed_type (GET_MODE_PRECISION (mode));
852 aarch64_simd_intOI_type_node = AARCH64_BUILD_SIGNED_TYPE (OImode);
f9d53c27
TB		 853 aarch64_simd_intCI_type_node = AARCH64_BUILD_SIGNED_TYPE (CImode);
854 aarch64_simd_intXI_type_node = AARCH64_BUILD_SIGNED_TYPE (XImode);
855#undef AARCH64_BUILD_SIGNED_TYPE
856
f9d53c27
TB		 857 tdecl = add_builtin_type
858 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node);
859 TYPE_NAME (aarch64_simd_intOI_type_node) = tdecl;
860 tdecl = add_builtin_type
861 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node);
862 TYPE_NAME (aarch64_simd_intCI_type_node) = tdecl;
863 tdecl = add_builtin_type
864 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node);
865 TYPE_NAME (aarch64_simd_intXI_type_node) = tdecl;
866}
867
868static void
869aarch64_init_simd_builtin_scalar_types (void)
870{
871 /* Define typedefs for all the standard scalar types. */
872 (*lang_hooks.types.register_builtin_type) (intQI_type_node,
43e9d192 873 "__builtin_aarch64_simd_qi");
f9d53c27 874 (*lang_hooks.types.register_builtin_type) (intHI_type_node,
43e9d192 875 "__builtin_aarch64_simd_hi");
7c369485
AL		 876 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node,
877 "__builtin_aarch64_simd_hf");
f9d53c27 878 (*lang_hooks.types.register_builtin_type) (intSI_type_node,
43e9d192 879 "__builtin_aarch64_simd_si");
f9d53c27 880 (*lang_hooks.types.register_builtin_type) (float_type_node,
43e9d192 881 "__builtin_aarch64_simd_sf");
f9d53c27 882 (*lang_hooks.types.register_builtin_type) (intDI_type_node,
43e9d192 883 "__builtin_aarch64_simd_di");
f9d53c27 884 (*lang_hooks.types.register_builtin_type) (double_type_node,
43e9d192 885 "__builtin_aarch64_simd_df");
f9d53c27 886 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
43e9d192 887 "__builtin_aarch64_simd_poly8");
f9d53c27 888 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
43e9d192 889 "__builtin_aarch64_simd_poly16");
f9d53c27 890 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
7baa225d 891 "__builtin_aarch64_simd_poly64");
f9d53c27 892 (*lang_hooks.types.register_builtin_type) (unsigned_intTI_type_node,
7baa225d 893 "__builtin_aarch64_simd_poly128");
f9d53c27 894 (*lang_hooks.types.register_builtin_type) (intTI_type_node,
43e9d192 895 "__builtin_aarch64_simd_ti");
e603cd43
MI		 896 (*lang_hooks.types.register_builtin_type) (aarch64_bf16_type_node,
897 "__builtin_aarch64_simd_bf");
b5828b4b 898 /* Unsigned integer types for various mode sizes. */
f9d53c27 899 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
b5828b4b 900 "__builtin_aarch64_simd_uqi");
f9d53c27 901 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
b5828b4b 902 "__builtin_aarch64_simd_uhi");
f9d53c27 903 (*lang_hooks.types.register_builtin_type) (unsigned_intSI_type_node,
b5828b4b 904 "__builtin_aarch64_simd_usi");
f9d53c27 905 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
b5828b4b 906 "__builtin_aarch64_simd_udi");
f9d53c27
TB		 907}
908
072a8b8f 909/* Return a set of FLAG_* flags that describe what the function could do,
910 taking the command-line flags into account. */
911static unsigned int
912aarch64_call_properties (aarch64_simd_builtin_datum *d)
913{
914 unsigned int flags = d->flags;
072a8b8f 915
35ffd4d1 916 if (!(flags & FLAG_AUTO_FP) && FLOAT_MODE_P (d->mode))
917 flags |= FLAG_FP;
072a8b8f 918
919 /* -fno-trapping-math means that we can assume any FP exceptions
920 are not user-visible. */
921 if (!flag_trapping_math)
922 flags &= ~FLAG_RAISE_FP_EXCEPTIONS;
923
924 return flags;
925}
926
927/* Return true if calls to the function could modify some form of
928 global state. */
929static bool
930aarch64_modifies_global_state_p (aarch64_simd_builtin_datum *d)
931{
932 unsigned int flags = aarch64_call_properties (d);
933
934 if (flags & FLAG_RAISE_FP_EXCEPTIONS)
935 return true;
936
937 if (flags & FLAG_PREFETCH_MEMORY)
938 return true;
939
940 return flags & FLAG_WRITE_MEMORY;
941}
942
943/* Return true if calls to the function could read some form of
944 global state. */
945static bool
946aarch64_reads_global_state_p (aarch64_simd_builtin_datum *d)
947{
948 unsigned int flags = aarch64_call_properties (d);
949
950 if (flags & FLAG_READ_FPCR)
951 return true;
952
953 return flags & FLAG_READ_MEMORY;
954}
955
956/* Return true if calls to the function could raise a signal. */
957static bool
958aarch64_could_trap_p (aarch64_simd_builtin_datum *d)
959{
960 unsigned int flags = aarch64_call_properties (d);
961
962 if (flags & FLAG_RAISE_FP_EXCEPTIONS)
963 return true;
964
965 if (flags & (FLAG_READ_MEMORY | FLAG_WRITE_MEMORY))
966 return true;
967
968 return false;
969}
970
971/* Add attribute NAME to ATTRS. */
972static tree
973aarch64_add_attribute (const char *name, tree attrs)
974{
975 return tree_cons (get_identifier (name), NULL_TREE, attrs);
976}
977
978/* Return the appropriate function attributes. */
979static tree
980aarch64_get_attributes (aarch64_simd_builtin_datum *d)
981{
982 tree attrs = NULL_TREE;
983
984 if (!aarch64_modifies_global_state_p (d))
985 {
986 if (aarch64_reads_global_state_p (d))
987 attrs = aarch64_add_attribute ("pure", attrs);
988 else
989 attrs = aarch64_add_attribute ("const", attrs);
990 }
991
992 if (!flag_non_call_exceptions || !aarch64_could_trap_p (d))
993 attrs = aarch64_add_attribute ("nothrow", attrs);
994
995 return aarch64_add_attribute ("leaf", attrs);
996}
997
e95a988a
KT		 998static bool aarch64_simd_builtins_initialized_p = false;
999
9d63f43b
TC		 1000/* Due to the architecture not providing lane variant of the lane instructions
1001 for fcmla we can't use the standard simd builtin expansion code, but we
1002 still want the majority of the validation that would normally be done. */
1003
1004void
1005aarch64_init_fcmla_laneq_builtins (void)
1006{
1007 unsigned int i = 0;
1008
1009 for (i = 0; i < ARRAY_SIZE (aarch64_fcmla_lane_builtin_data); ++i)
1010 {
1011 aarch64_fcmla_laneq_builtin_datum* d
1012 = &aarch64_fcmla_lane_builtin_data[i];
1013 tree argtype = aarch64_lookup_simd_builtin_type (d->mode, qualifier_none);
1014 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
1015 tree quadtype
1016 = aarch64_lookup_simd_builtin_type (quadmode, qualifier_none);
1017 tree lanetype
1018 = aarch64_simd_builtin_std_type (SImode, qualifier_lane_pair_index);
1019 tree ftype = build_function_type_list (argtype, argtype, argtype,
1020 quadtype, lanetype, NULL_TREE);
6d4d616a 1021 tree fndecl = aarch64_general_add_builtin (d->name, ftype, d->fcode);
9d63f43b
TC		 1022
1023 aarch64_builtin_decls[d->fcode] = fndecl;
1024 }
1025}
1026
e95a988a 1027void
f9d53c27
TB		 1028aarch64_init_simd_builtins (void)
1029{
661fce82 1030 unsigned int i, fcode = AARCH64_SIMD_PATTERN_START;
f9d53c27 1031
e95a988a
KT		 1032 if (aarch64_simd_builtins_initialized_p)
1033 return;
1034
1035 aarch64_simd_builtins_initialized_p = true;
1036
f9d53c27 1037 aarch64_init_simd_builtin_types ();
43e9d192 1038
f9d53c27
TB		 1039 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
1040 Therefore we need to preserve the old __builtin scalar types. It can be
1041 removed once all the intrinsics become strongly typed using the qualifier
1042 system. */
1043 aarch64_init_simd_builtin_scalar_types ();
1044
661fce82 1045 tree lane_check_fpr = build_function_type_list (void_type_node,
9c4f25cc
AP		 1046 size_type_node,
1047 size_type_node,
661fce82
AL		 1048 intSI_type_node,
1049 NULL);
6d4d616a
RS		 1050 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK]
1051 = aarch64_general_add_builtin ("__builtin_aarch64_im_lane_boundsi",
1052 lane_check_fpr,
1053 AARCH64_SIMD_BUILTIN_LANE_CHECK);
661fce82 1054
342be7f7 1055 for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++)
43e9d192 1056 {
b5828b4b 1057 bool print_type_signature_p = false;
cae83731 1058 char type_signature[SIMD_MAX_BUILTIN_ARGS + 1] = { 0 };
43e9d192 1059 aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i];
342be7f7
JG		 1060 char namebuf[60];
1061 tree ftype = NULL;
119103ca 1062 tree fndecl = NULL;
342be7f7 1063
342be7f7 1064 d->fcode = fcode;
43e9d192 1065
b5828b4b
JG		 1066 /* We must track two variables here. op_num is
1067 the operand number as in the RTL pattern. This is
1068 required to access the mode (e.g. V4SF mode) of the
1069 argument, from which the base type can be derived.
1070 arg_num is an index in to the qualifiers data, which
1071 gives qualifiers to the type (e.g. const unsigned).
1072 The reason these two variables may differ by one is the
1073 void return type. While all return types take the 0th entry
1074 in the qualifiers array, there is no operand for them in the
1075 RTL pattern. */
1076 int op_num = insn_data[d->code].n_operands - 1;
1077 int arg_num = d->qualifiers[0] & qualifier_void
1078 ? op_num + 1
1079 : op_num;
1080 tree return_type = void_type_node, args = void_list_node;
1081 tree eltype;
1082
1083 /* Build a function type directly from the insn_data for this
1084 builtin. The build_function_type () function takes care of
1085 removing duplicates for us. */
1086 for (; op_num >= 0; arg_num--, op_num--)
43e9d192 1087 {
ef4bddc2 1088 machine_mode op_mode = insn_data[d->code].operand[op_num].mode;
b5828b4b 1089 enum aarch64_type_qualifiers qualifiers = d->qualifiers[arg_num];
43e9d192 1090
b5828b4b
JG		 1091 if (qualifiers & qualifier_unsigned)
1092 {
9fd2074d 1093 type_signature[op_num] = 'u';
b5828b4b
JG		 1094 print_type_signature_p = true;
1095 }
6db1ec94
JG		 1096 else if (qualifiers & qualifier_poly)
1097 {
9fd2074d 1098 type_signature[op_num] = 'p';
6db1ec94
JG		 1099 print_type_signature_p = true;
1100 }
b5828b4b 1101 else
9fd2074d 1102 type_signature[op_num] = 's';
b5828b4b
JG		 1103
1104 /* Skip an internal operand for vget_{low, high}. */
1105 if (qualifiers & qualifier_internal)
1106 continue;
1107
1108 /* Some builtins have different user-facing types
1109 for certain arguments, encoded in d->mode. */
1110 if (qualifiers & qualifier_map_mode)
bc5e395d 1111 op_mode = d->mode;
b5828b4b
JG		 1112
1113 /* For pointers, we want a pointer to the basic type
1114 of the vector. */
1115 if (qualifiers & qualifier_pointer && VECTOR_MODE_P (op_mode))
1116 op_mode = GET_MODE_INNER (op_mode);
1117
f9d53c27
TB		 1118 eltype = aarch64_simd_builtin_type
1119 (op_mode,
1120 (qualifiers & qualifier_unsigned) != 0,
1121 (qualifiers & qualifier_poly) != 0);
1122 gcc_assert (eltype != NULL);
b5828b4b
JG		 1123
1124 /* Add qualifiers. */
1125 if (qualifiers & qualifier_const)
1126 eltype = build_qualified_type (eltype, TYPE_QUAL_CONST);
1127
1128 if (qualifiers & qualifier_pointer)
1129 eltype = build_pointer_type (eltype);
1130
1131 /* If we have reached arg_num == 0, we are at a non-void
1132 return type. Otherwise, we are still processing
1133 arguments. */
1134 if (arg_num == 0)
1135 return_type = eltype;
1136 else
1137 args = tree_cons (NULL_TREE, eltype, args);
1138 }
342be7f7 1139
b5828b4b 1140 ftype = build_function_type (return_type, args);
43e9d192 1141
342be7f7 1142 gcc_assert (ftype != NULL);
43e9d192 1143
b5828b4b 1144 if (print_type_signature_p)
bc5e395d
JG		 1145 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s_%s",
1146 d->name, type_signature);
b5828b4b 1147 else
bc5e395d
JG		 1148 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s",
1149 d->name);
43e9d192 1150
072a8b8f 1151 tree attrs = aarch64_get_attributes (d);
1152
1153 fndecl = aarch64_general_add_builtin (namebuf, ftype, fcode, attrs);
119103ca 1154 aarch64_builtin_decls[fcode] = fndecl;
43e9d192 1155 }
280d970b
TC		 1156
1157 /* Initialize the remaining fcmla_laneq intrinsics. */
1158 aarch64_init_fcmla_laneq_builtins ();
43e9d192
IB		 1159}
1160
5d357f26
KT		 1161static void
1162aarch64_init_crc32_builtins ()
1163{
f9d53c27 1164 tree usi_type = aarch64_simd_builtin_std_type (SImode, qualifier_unsigned);
5d357f26
KT		 1165 unsigned int i = 0;
1166
1167 for (i = 0; i < ARRAY_SIZE (aarch64_crc_builtin_data); ++i)
1168 {
1169 aarch64_crc_builtin_datum* d = &aarch64_crc_builtin_data[i];
f9d53c27
TB		 1170 tree argtype = aarch64_simd_builtin_std_type (d->mode,
1171 qualifier_unsigned);
5d357f26 1172 tree ftype = build_function_type_list (usi_type, usi_type, argtype, NULL_TREE);
6d4d616a 1173 tree fndecl = aarch64_general_add_builtin (d->name, ftype, d->fcode);
5d357f26
KT		 1174
1175 aarch64_builtin_decls[d->fcode] = fndecl;
1176 }
1177}
1178
a6fc00da
BH		 1179/* Add builtins for reciprocal square root. */
1180
1181void
1182aarch64_init_builtin_rsqrt (void)
1183{
1184 tree fndecl = NULL;
1185 tree ftype = NULL;
1186
1187 tree V2SF_type_node = build_vector_type (float_type_node, 2);
1188 tree V2DF_type_node = build_vector_type (double_type_node, 2);
1189 tree V4SF_type_node = build_vector_type (float_type_node, 4);
1190
1191 struct builtin_decls_data
1192 {
1193 tree type_node;
1194 const char *builtin_name;
1195 int function_code;
1196 };
1197
1198 builtin_decls_data bdda[] =
1199 {
1200 { double_type_node, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF },
1201 { float_type_node, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF },
1202 { V2DF_type_node, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF },
1203 { V2SF_type_node, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF },
1204 { V4SF_type_node, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF }
1205 };
1206
1207 builtin_decls_data *bdd = bdda;
1208 builtin_decls_data *bdd_end = bdd + (sizeof (bdda) / sizeof (builtin_decls_data));
1209
1210 for (; bdd < bdd_end; bdd++)
1211 {
1212 ftype = build_function_type_list (bdd->type_node, bdd->type_node, NULL_TREE);
6d4d616a
RS		 1213 fndecl = aarch64_general_add_builtin (bdd->builtin_name,
1214 ftype, bdd->function_code);
a6fc00da
BH		 1215 aarch64_builtin_decls[bdd->function_code] = fndecl;
1216 }
1217}
1218
1b62ed4f
JG		 1219/* Initialize the backend types that support the user-visible __fp16
1220 type, also initialize a pointer to that type, to be used when
1221 forming HFAs. */
1222
1223static void
1224aarch64_init_fp16_types (void)
1225{
1226 aarch64_fp16_type_node = make_node (REAL_TYPE);
1227 TYPE_PRECISION (aarch64_fp16_type_node) = 16;
1228 layout_type (aarch64_fp16_type_node);
1229
1230 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node, "__fp16");
1231 aarch64_fp16_ptr_type_node = build_pointer_type (aarch64_fp16_type_node);
1232}
1233
abbe1ed2
SMW		 1234/* Initialize the backend REAL_TYPE type supporting bfloat types. */
1235static void
1236aarch64_init_bf16_types (void)
1237{
1238 aarch64_bf16_type_node = make_node (REAL_TYPE);
1239 TYPE_PRECISION (aarch64_bf16_type_node) = 16;
1240 SET_TYPE_MODE (aarch64_bf16_type_node, BFmode);
1241 layout_type (aarch64_bf16_type_node);
1242
1243 lang_hooks.types.register_builtin_type (aarch64_bf16_type_node, "__bf16");
1244 aarch64_bf16_ptr_type_node = build_pointer_type (aarch64_bf16_type_node);
1245}
1246
312492bd
JW		 1247/* Pointer authentication builtins that will become NOP on legacy platform.
1248 Currently, these builtins are for internal use only (libgcc EH unwinder). */
1249
1250void
1251aarch64_init_pauth_hint_builtins (void)
1252{
1253 /* Pointer Authentication builtins. */
1254 tree ftype_pointer_auth
1255 = build_function_type_list (ptr_type_node, ptr_type_node,
1256 unsigned_intDI_type_node, NULL_TREE);
1257 tree ftype_pointer_strip
1258 = build_function_type_list (ptr_type_node, ptr_type_node, NULL_TREE);
1259
1260 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIA1716]
6d4d616a
RS		 1261 = aarch64_general_add_builtin ("__builtin_aarch64_autia1716",
1262 ftype_pointer_auth,
1263 AARCH64_PAUTH_BUILTIN_AUTIA1716);
312492bd 1264 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIA1716]
6d4d616a
RS		 1265 = aarch64_general_add_builtin ("__builtin_aarch64_pacia1716",
1266 ftype_pointer_auth,
1267 AARCH64_PAUTH_BUILTIN_PACIA1716);
8fc16d72 1268 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIB1716]
6d4d616a
RS		 1269 = aarch64_general_add_builtin ("__builtin_aarch64_autib1716",
1270 ftype_pointer_auth,
1271 AARCH64_PAUTH_BUILTIN_AUTIB1716);
8fc16d72 1272 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIB1716]
6d4d616a
RS		 1273 = aarch64_general_add_builtin ("__builtin_aarch64_pacib1716",
1274 ftype_pointer_auth,
1275 AARCH64_PAUTH_BUILTIN_PACIB1716);
312492bd 1276 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_XPACLRI]
6d4d616a
RS		 1277 = aarch64_general_add_builtin ("__builtin_aarch64_xpaclri",
1278 ftype_pointer_strip,
1279 AARCH64_PAUTH_BUILTIN_XPACLRI);
312492bd
JW		 1280}
1281
89626179
SD		 1282/* Initialize the transactional memory extension (TME) builtins. */
1283static void
1284aarch64_init_tme_builtins (void)
1285{
1286 tree ftype_uint64_void
1287 = build_function_type_list (uint64_type_node, NULL);
1288 tree ftype_void_void
1289 = build_function_type_list (void_type_node, NULL);
1290 tree ftype_void_uint64
1291 = build_function_type_list (void_type_node, uint64_type_node, NULL);
1292
1293 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TSTART]
6d4d616a
RS		 1294 = aarch64_general_add_builtin ("__builtin_aarch64_tstart",
1295 ftype_uint64_void,
1296 AARCH64_TME_BUILTIN_TSTART);
89626179 1297 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TTEST]
6d4d616a
RS		 1298 = aarch64_general_add_builtin ("__builtin_aarch64_ttest",
1299 ftype_uint64_void,
1300 AARCH64_TME_BUILTIN_TTEST);
89626179 1301 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCOMMIT]
6d4d616a
RS		 1302 = aarch64_general_add_builtin ("__builtin_aarch64_tcommit",
1303 ftype_void_void,
1304 AARCH64_TME_BUILTIN_TCOMMIT);
89626179 1305 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCANCEL]
6d4d616a
RS		 1306 = aarch64_general_add_builtin ("__builtin_aarch64_tcancel",
1307 ftype_void_uint64,
1308 AARCH64_TME_BUILTIN_TCANCEL);
89626179
SD		 1309}
1310
c5dc215d
KT		 1311/* Add builtins for Random Number instructions. */
1312
1313static void
1314aarch64_init_rng_builtins (void)
1315{
1316 tree unsigned_ptr_type = build_pointer_type (unsigned_intDI_type_node);
1317 tree ftype
1318 = build_function_type_list (integer_type_node, unsigned_ptr_type, NULL);
1319 aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDR]
1320 = aarch64_general_add_builtin ("__builtin_aarch64_rndr", ftype,
1321 AARCH64_BUILTIN_RNG_RNDR);
1322 aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDRRS]
1323 = aarch64_general_add_builtin ("__builtin_aarch64_rndrrs", ftype,
1324 AARCH64_BUILTIN_RNG_RNDRRS);
1325}
1326
ef01e6bb
DZ		 1327/* Initialize the memory tagging extension (MTE) builtins. */
1328struct
1329{
1330 tree ftype;
1331 enum insn_code icode;
1332} aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_END -
1333 AARCH64_MEMTAG_BUILTIN_START - 1];
1334
1335static void
1336aarch64_init_memtag_builtins (void)
1337{
1338 tree fntype = NULL;
1339
1340#define AARCH64_INIT_MEMTAG_BUILTINS_DECL(F, N, I, T) \
1341 aarch64_builtin_decls[AARCH64_MEMTAG_BUILTIN_##F] \
1342 = aarch64_general_add_builtin ("__builtin_aarch64_memtag_"#N, \
1343 T, AARCH64_MEMTAG_BUILTIN_##F); \
1344 aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_##F - \
1345 AARCH64_MEMTAG_BUILTIN_START - 1] = \
1346 {T, CODE_FOR_##I};
1347
1348 fntype = build_function_type_list (ptr_type_node, ptr_type_node,
1349 uint64_type_node, NULL);
1350 AARCH64_INIT_MEMTAG_BUILTINS_DECL (IRG, irg, irg, fntype);
1351
1352 fntype = build_function_type_list (uint64_type_node, ptr_type_node,
1353 uint64_type_node, NULL);
1354 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GMI, gmi, gmi, fntype);
1355
1356 fntype = build_function_type_list (ptrdiff_type_node, ptr_type_node,
1357 ptr_type_node, NULL);
1358 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SUBP, subp, subp, fntype);
1359
1360 fntype = build_function_type_list (ptr_type_node, ptr_type_node,
1361 unsigned_type_node, NULL);
1362 AARCH64_INIT_MEMTAG_BUILTINS_DECL (INC_TAG, inc_tag, addg, fntype);
1363
1364 fntype = build_function_type_list (void_type_node, ptr_type_node, NULL);
1365 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SET_TAG, set_tag, stg, fntype);
1366
1367 fntype = build_function_type_list (ptr_type_node, ptr_type_node, NULL);
1368 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GET_TAG, get_tag, ldg, fntype);
1369
1370#undef AARCH64_INIT_MEMTAG_BUILTINS_DECL
1371}
c5dc215d 1372
0d7e5fa6 1373/* Initialize fpsr fpcr getters and setters. */
c5dc215d 1374
0d7e5fa6
AC		 1375static void
1376aarch64_init_fpsr_fpcr_builtins (void)
43e9d192 1377{
0d7e5fa6 1378 tree ftype_set
aa87aced 1379 = build_function_type_list (void_type_node, unsigned_type_node, NULL);
0d7e5fa6 1380 tree ftype_get
aa87aced
KV		 1381 = build_function_type_list (unsigned_type_node, NULL);
1382
1383 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR]
6d4d616a 1384 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr",
0d7e5fa6 1385 ftype_get,
6d4d616a 1386 AARCH64_BUILTIN_GET_FPCR);
aa87aced 1387 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR]
6d4d616a 1388 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr",
0d7e5fa6 1389 ftype_set,
6d4d616a 1390 AARCH64_BUILTIN_SET_FPCR);
aa87aced 1391 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR]
6d4d616a 1392 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr",
0d7e5fa6 1393 ftype_get,
6d4d616a 1394 AARCH64_BUILTIN_GET_FPSR);
aa87aced 1395 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR]
6d4d616a 1396 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr",
0d7e5fa6 1397 ftype_set,
6d4d616a 1398 AARCH64_BUILTIN_SET_FPSR);
aa87aced 1399
0d7e5fa6
AC		 1400 ftype_set
1401 = build_function_type_list (void_type_node, long_long_unsigned_type_node,
1402 NULL);
1403 ftype_get
1404 = build_function_type_list (long_long_unsigned_type_node, NULL);
1405
1406 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR64]
1407 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr64",
1408 ftype_get,
1409 AARCH64_BUILTIN_GET_FPCR64);
1410 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR64]
1411 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr64",
1412 ftype_set,
1413 AARCH64_BUILTIN_SET_FPCR64);
1414 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR64]
1415 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr64",
1416 ftype_get,
1417 AARCH64_BUILTIN_GET_FPSR64);
1418 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR64]
1419 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr64",
1420 ftype_set,
1421 AARCH64_BUILTIN_SET_FPSR64);
1422}
1423
1424/* Initialize all builtins in the AARCH64_BUILTIN_GENERAL group. */
1425
1426void
1427aarch64_general_init_builtins (void)
1428{
1429 aarch64_init_fpsr_fpcr_builtins ();
1430
1b62ed4f 1431 aarch64_init_fp16_types ();
c2ec330c 1432
abbe1ed2
SMW		 1433 aarch64_init_bf16_types ();
1434
342be7f7 1435 if (TARGET_SIMD)
280d970b 1436 aarch64_init_simd_builtins ();
e95a988a
KT		 1437
1438 aarch64_init_crc32_builtins ();
a6fc00da 1439 aarch64_init_builtin_rsqrt ();
c5dc215d 1440 aarch64_init_rng_builtins ();
312492bd 1441
e1d5d19e
KT		 1442 tree ftype_jcvt
1443 = build_function_type_list (intSI_type_node, double_type_node, NULL);
1444 aarch64_builtin_decls[AARCH64_JSCVT]
6d4d616a
RS		 1445 = aarch64_general_add_builtin ("__builtin_aarch64_jcvtzs", ftype_jcvt,
1446 AARCH64_JSCVT);
e1d5d19e 1447
a876231c
JW		 1448 /* Initialize pointer authentication builtins which are backed by instructions
1449 in NOP encoding space.
1450
1451 NOTE: these builtins are supposed to be used by libgcc unwinder only, as
1452 there is no support on return address signing under ILP32, we don't
1453 register them. */
1454 if (!TARGET_ILP32)
1455 aarch64_init_pauth_hint_builtins ();
89626179
SD		 1456
1457 if (TARGET_TME)
1458 aarch64_init_tme_builtins ();
ef01e6bb
DZ		 1459
1460 if (TARGET_MEMTAG)
1461 aarch64_init_memtag_builtins ();
43e9d192
IB		 1462}
1463
6d4d616a 1464/* Implement TARGET_BUILTIN_DECL for the AARCH64_BUILTIN_GENERAL group. */
119103ca 1465tree
6d4d616a 1466aarch64_general_builtin_decl (unsigned code, bool)
119103ca
JG		 1467{
1468 if (code >= AARCH64_BUILTIN_MAX)
1469 return error_mark_node;
1470
1471 return aarch64_builtin_decls[code];
1472}
1473
43e9d192
IB		 1474typedef enum
1475{
1476 SIMD_ARG_COPY_TO_REG,
1477 SIMD_ARG_CONSTANT,
2a49c16d 1478 SIMD_ARG_LANE_INDEX,
4d0a0237 1479 SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX,
9d63f43b 1480 SIMD_ARG_LANE_PAIR_INDEX,
8c197c85 1481 SIMD_ARG_LANE_QUADTUP_INDEX,
43e9d192
IB		 1482 SIMD_ARG_STOP
1483} builtin_simd_arg;
1484
e95a988a 1485
43e9d192
IB		 1486static rtx
1487aarch64_simd_expand_args (rtx target, int icode, int have_retval,
4d0a0237 1488 tree exp, builtin_simd_arg *args,
b8506a8a 1489 machine_mode builtin_mode)
43e9d192 1490{
43e9d192 1491 rtx pat;
d9e80f49
AL		 1492 rtx op[SIMD_MAX_BUILTIN_ARGS + 1]; /* First element for result operand. */
1493 int opc = 0;
1494
1495 if (have_retval)
1496 {
1497 machine_mode tmode = insn_data[icode].operand[0].mode;
1498 if (!target
43e9d192 1499 || GET_MODE (target) != tmode
d9e80f49
AL		 1500 || !(*insn_data[icode].operand[0].predicate) (target, tmode))
1501 target = gen_reg_rtx (tmode);
1502 op[opc++] = target;
1503 }
43e9d192 1504
43e9d192
IB		 1505 for (;;)
1506 {
d9e80f49 1507 builtin_simd_arg thisarg = args[opc - have_retval];
43e9d192
IB		 1508
1509 if (thisarg == SIMD_ARG_STOP)
1510 break;
1511 else
1512 {
d9e80f49 1513 tree arg = CALL_EXPR_ARG (exp, opc - have_retval);
b8506a8a 1514 machine_mode mode = insn_data[icode].operand[opc].mode;
d9e80f49 1515 op[opc] = expand_normal (arg);
43e9d192
IB		 1516
1517 switch (thisarg)
1518 {
1519 case SIMD_ARG_COPY_TO_REG:
d9e80f49
AL		 1520 if (POINTER_TYPE_P (TREE_TYPE (arg)))
1521 op[opc] = convert_memory_address (Pmode, op[opc]);
1522 /*gcc_assert (GET_MODE (op[opc]) == mode); */
1523 if (!(*insn_data[icode].operand[opc].predicate)
1524 (op[opc], mode))
1525 op[opc] = copy_to_mode_reg (mode, op[opc]);
43e9d192
IB		 1526 break;
1527
4d0a0237
CB		 1528 case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX:
1529 gcc_assert (opc > 1);
1530 if (CONST_INT_P (op[opc]))
1531 {
6a70badb
RS		 1532 unsigned int nunits
1533 = GET_MODE_NUNITS (builtin_mode).to_constant ();
1534 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
4d0a0237 1535 /* Keep to GCC-vector-extension lane indices in the RTL. */
7ac29c0f
RS		 1536 op[opc] = aarch64_endian_lane_rtx (builtin_mode,
1537 INTVAL (op[opc]));
4d0a0237
CB		 1538 }
1539 goto constant_arg;
1540
2a49c16d
AL		 1541 case SIMD_ARG_LANE_INDEX:
1542 /* Must be a previous operand into which this is an index. */
d9e80f49
AL		 1543 gcc_assert (opc > 0);
1544 if (CONST_INT_P (op[opc]))
2a49c16d 1545 {
d9e80f49 1546 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
6a70badb
RS		 1547 unsigned int nunits
1548 = GET_MODE_NUNITS (vmode).to_constant ();
1549 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
2a49c16d 1550 /* Keep to GCC-vector-extension lane indices in the RTL. */
7ac29c0f 1551 op[opc] = aarch64_endian_lane_rtx (vmode, INTVAL (op[opc]));
2a49c16d 1552 }
9d63f43b
TC		 1553 /* If the lane index isn't a constant then error out. */
1554 goto constant_arg;
1555
1556 case SIMD_ARG_LANE_PAIR_INDEX:
1557 /* Must be a previous operand into which this is an index and
1558 index is restricted to nunits / 2. */
1559 gcc_assert (opc > 0);
1560 if (CONST_INT_P (op[opc]))
1561 {
1562 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1563 unsigned int nunits
1564 = GET_MODE_NUNITS (vmode).to_constant ();
1565 aarch64_simd_lane_bounds (op[opc], 0, nunits / 2, exp);
1566 /* Keep to GCC-vector-extension lane indices in the RTL. */
33b5a38c
TC		 1567 int lane = INTVAL (op[opc]);
1568 op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane),
1569 SImode);
9d63f43b 1570 }
8c197c85
SMW		 1571 /* If the lane index isn't a constant then error out. */
1572 goto constant_arg;
1573 case SIMD_ARG_LANE_QUADTUP_INDEX:
1574 /* Must be a previous operand into which this is an index and
1575 index is restricted to nunits / 4. */
1576 gcc_assert (opc > 0);
1577 if (CONST_INT_P (op[opc]))
1578 {
1579 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1580 unsigned int nunits
1581 = GET_MODE_NUNITS (vmode).to_constant ();
1582 aarch64_simd_lane_bounds (op[opc], 0, nunits / 4, exp);
1583 /* Keep to GCC-vector-extension lane indices in the RTL. */
1584 int lane = INTVAL (op[opc]);
1585 op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 4, lane),
1586 SImode);
1587 }
1588 /* If the lane index isn't a constant then error out. */
1589 goto constant_arg;
43e9d192 1590 case SIMD_ARG_CONSTANT:
4d0a0237 1591constant_arg:
d9e80f49
AL		 1592 if (!(*insn_data[icode].operand[opc].predicate)
1593 (op[opc], mode))
d5a29419 1594 {
fca051af
AL		 1595 error ("%Kargument %d must be a constant immediate",
1596 exp, opc + 1 - have_retval);
d5a29419
KT		 1597 return const0_rtx;
1598 }
43e9d192
IB		 1599 break;
1600
1601 case SIMD_ARG_STOP:
1602 gcc_unreachable ();
1603 }
1604
d9e80f49 1605 opc++;
43e9d192
IB		 1606 }
1607 }
1608
d9e80f49
AL		 1609 switch (opc)
1610 {
1611 case 1:
1612 pat = GEN_FCN (icode) (op[0]);
1613 break;
43e9d192 1614
d9e80f49
AL		 1615 case 2:
1616 pat = GEN_FCN (icode) (op[0], op[1]);
1617 break;
43e9d192 1618
d9e80f49
AL		 1619 case 3:
1620 pat = GEN_FCN (icode) (op[0], op[1], op[2]);
1621 break;
43e9d192 1622
d9e80f49
AL		 1623 case 4:
1624 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]);
1625 break;
43e9d192 1626
d9e80f49
AL		 1627 case 5:
1628 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]);
1629 break;
43e9d192 1630
d9e80f49
AL		 1631 case 6:
1632 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5]);
1633 break;
43e9d192 1634
d9e80f49
AL		 1635 default:
1636 gcc_unreachable ();
1637 }
43e9d192
IB		 1638
1639 if (!pat)
d5a29419 1640 return NULL_RTX;
43e9d192
IB		 1641
1642 emit_insn (pat);
1643
1644 return target;
1645}
1646
1647/* Expand an AArch64 AdvSIMD builtin(intrinsic). */
1648rtx
1649aarch64_simd_expand_builtin (int fcode, tree exp, rtx target)
1650{
661fce82
AL		 1651 if (fcode == AARCH64_SIMD_BUILTIN_LANE_CHECK)
1652 {
9c4f25cc
AP		 1653 rtx totalsize = expand_normal (CALL_EXPR_ARG (exp, 0));
1654 rtx elementsize = expand_normal (CALL_EXPR_ARG (exp, 1));
1655 if (CONST_INT_P (totalsize) && CONST_INT_P (elementsize)
1656 && UINTVAL (elementsize) != 0
1657 && UINTVAL (totalsize) != 0)
1658 {
1659 rtx lane_idx = expand_normal (CALL_EXPR_ARG (exp, 2));
1660 if (CONST_INT_P (lane_idx))
1661 aarch64_simd_lane_bounds (lane_idx, 0,
1662 UINTVAL (totalsize)
1663 / UINTVAL (elementsize),
1664 exp);
1665 else
1666 error ("%Klane index must be a constant immediate", exp);
1667 }
661fce82 1668 else
9c4f25cc 1669 error ("%Ktotal size and element size must be a non-zero constant immediate", exp);
661fce82
AL		 1670 /* Don't generate any RTL. */
1671 return const0_rtx;
1672 }
342be7f7 1673 aarch64_simd_builtin_datum *d =
661fce82 1674 &aarch64_simd_builtin_data[fcode - AARCH64_SIMD_PATTERN_START];
342be7f7 1675 enum insn_code icode = d->code;
0ff2bf46 1676 builtin_simd_arg args[SIMD_MAX_BUILTIN_ARGS + 1];
b5828b4b
JG		 1677 int num_args = insn_data[d->code].n_operands;
1678 int is_void = 0;
1679 int k;
43e9d192 1680
b5828b4b 1681 is_void = !!(d->qualifiers[0] & qualifier_void);
43e9d192 1682
b5828b4b
JG		 1683 num_args += is_void;
1684
1685 for (k = 1; k < num_args; k++)
1686 {
1687 /* We have four arrays of data, each indexed in a different fashion.
1688 qualifiers - element 0 always describes the function return type.
1689 operands - element 0 is either the operand for return value (if
1690 the function has a non-void return type) or the operand for the
1691 first argument.
1692 expr_args - element 0 always holds the first argument.
1693 args - element 0 is always used for the return type. */
1694 int qualifiers_k = k;
1695 int operands_k = k - is_void;
1696 int expr_args_k = k - 1;
1697
2a49c16d
AL		 1698 if (d->qualifiers[qualifiers_k] & qualifier_lane_index)
1699 args[k] = SIMD_ARG_LANE_INDEX;
9d63f43b
TC		 1700 else if (d->qualifiers[qualifiers_k] & qualifier_lane_pair_index)
1701 args[k] = SIMD_ARG_LANE_PAIR_INDEX;
8c197c85
SMW		 1702 else if (d->qualifiers[qualifiers_k] & qualifier_lane_quadtup_index)
1703 args[k] = SIMD_ARG_LANE_QUADTUP_INDEX;
4d0a0237
CB		 1704 else if (d->qualifiers[qualifiers_k] & qualifier_struct_load_store_lane_index)
1705 args[k] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX;
2a49c16d 1706 else if (d->qualifiers[qualifiers_k] & qualifier_immediate)
b5828b4b
JG		 1707 args[k] = SIMD_ARG_CONSTANT;
1708 else if (d->qualifiers[qualifiers_k] & qualifier_maybe_immediate)
1709 {
1710 rtx arg
1711 = expand_normal (CALL_EXPR_ARG (exp,
1712 (expr_args_k)));
1713 /* Handle constants only if the predicate allows it. */
1714 bool op_const_int_p =
1715 (CONST_INT_P (arg)
1716 && (*insn_data[icode].operand[operands_k].predicate)
1717 (arg, insn_data[icode].operand[operands_k].mode));
1718 args[k] = op_const_int_p ? SIMD_ARG_CONSTANT : SIMD_ARG_COPY_TO_REG;
1719 }
1720 else
1721 args[k] = SIMD_ARG_COPY_TO_REG;
43e9d192 1722
43e9d192 1723 }
b5828b4b
JG		 1724 args[k] = SIMD_ARG_STOP;
1725
1726 /* The interface to aarch64_simd_expand_args expects a 0 if
1727 the function is void, and a 1 if it is not. */
1728 return aarch64_simd_expand_args
4d0a0237 1729 (target, icode, !is_void, exp, &args[1], d->mode);
43e9d192 1730}
342be7f7 1731
5d357f26
KT		 1732rtx
1733aarch64_crc32_expand_builtin (int fcode, tree exp, rtx target)
1734{
1735 rtx pat;
1736 aarch64_crc_builtin_datum *d
1737 = &aarch64_crc_builtin_data[fcode - (AARCH64_CRC32_BUILTIN_BASE + 1)];
1738 enum insn_code icode = d->icode;
1739 tree arg0 = CALL_EXPR_ARG (exp, 0);
1740 tree arg1 = CALL_EXPR_ARG (exp, 1);
1741 rtx op0 = expand_normal (arg0);
1742 rtx op1 = expand_normal (arg1);
ef4bddc2
RS		 1743 machine_mode tmode = insn_data[icode].operand[0].mode;
1744 machine_mode mode0 = insn_data[icode].operand[1].mode;
1745 machine_mode mode1 = insn_data[icode].operand[2].mode;
5d357f26
KT		 1746
1747 if (! target
1748 || GET_MODE (target) != tmode
1749 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
1750 target = gen_reg_rtx (tmode);
1751
1752 gcc_assert ((GET_MODE (op0) == mode0 || GET_MODE (op0) == VOIDmode)
1753 && (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode));
1754
1755 if (! (*insn_data[icode].operand[1].predicate) (op0, mode0))
1756 op0 = copy_to_mode_reg (mode0, op0);
1757 if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
1758 op1 = copy_to_mode_reg (mode1, op1);
1759
1760 pat = GEN_FCN (icode) (target, op0, op1);
d5a29419
KT		 1761 if (!pat)
1762 return NULL_RTX;
1763
5d357f26
KT		 1764 emit_insn (pat);
1765 return target;
1766}
1767
a6fc00da
BH		 1768/* Function to expand reciprocal square root builtins. */
1769
1770static rtx
1771aarch64_expand_builtin_rsqrt (int fcode, tree exp, rtx target)
1772{
1773 tree arg0 = CALL_EXPR_ARG (exp, 0);
1774 rtx op0 = expand_normal (arg0);
1775
1776 rtx (*gen) (rtx, rtx);
1777
1778 switch (fcode)
1779 {
1780 case AARCH64_BUILTIN_RSQRT_DF:
ee62a5a6 1781 gen = gen_rsqrtdf2;
a6fc00da
BH		 1782 break;
1783 case AARCH64_BUILTIN_RSQRT_SF:
ee62a5a6 1784 gen = gen_rsqrtsf2;
a6fc00da
BH		 1785 break;
1786 case AARCH64_BUILTIN_RSQRT_V2DF:
ee62a5a6 1787 gen = gen_rsqrtv2df2;
a6fc00da
BH		 1788 break;
1789 case AARCH64_BUILTIN_RSQRT_V2SF:
ee62a5a6 1790 gen = gen_rsqrtv2sf2;
a6fc00da
BH		 1791 break;
1792 case AARCH64_BUILTIN_RSQRT_V4SF:
ee62a5a6 1793 gen = gen_rsqrtv4sf2;
a6fc00da
BH		 1794 break;
1795 default: gcc_unreachable ();
1796 }
1797
1798 if (!target)
1799 target = gen_reg_rtx (GET_MODE (op0));
1800
1801 emit_insn (gen (target, op0));
1802
1803 return target;
1804}
1805
9d63f43b
TC		 1806/* Expand a FCMLA lane expression EXP with code FCODE and
1807 result going to TARGET if that is convenient. */
1808
1809rtx
1810aarch64_expand_fcmla_builtin (tree exp, rtx target, int fcode)
1811{
1812 int bcode = fcode - AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE - 1;
1813 aarch64_fcmla_laneq_builtin_datum* d
1814 = &aarch64_fcmla_lane_builtin_data[bcode];
1815 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
1816 rtx op0 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 0)));
1817 rtx op1 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 1)));
1818 rtx op2 = force_reg (quadmode, expand_normal (CALL_EXPR_ARG (exp, 2)));
1819 tree tmp = CALL_EXPR_ARG (exp, 3);
1820 rtx lane_idx = expand_expr (tmp, NULL_RTX, VOIDmode, EXPAND_INITIALIZER);
1821
1822 /* Validate that the lane index is a constant. */
1823 if (!CONST_INT_P (lane_idx))
1824 {
1825 error ("%Kargument %d must be a constant immediate", exp, 4);
1826 return const0_rtx;
1827 }
1828
1829 /* Validate that the index is within the expected range. */
1830 int nunits = GET_MODE_NUNITS (quadmode).to_constant ();
1831 aarch64_simd_lane_bounds (lane_idx, 0, nunits / 2, exp);
1832
9d63f43b
TC		 1833 /* Generate the correct register and mode. */
1834 int lane = INTVAL (lane_idx);
1835
1836 if (lane < nunits / 4)
33b5a38c
TC		 1837 op2 = simplify_gen_subreg (d->mode, op2, quadmode,
1838 subreg_lowpart_offset (d->mode, quadmode));
9d63f43b
TC		 1839 else
1840 {
1841 /* Select the upper 64 bits, either a V2SF or V4HF, this however
1842 is quite messy, as the operation required even though simple
1843 doesn't have a simple RTL pattern, and seems it's quite hard to
1844 define using a single RTL pattern. The target generic version
1845 gen_highpart_mode generates code that isn't optimal. */
1846 rtx temp1 = gen_reg_rtx (d->mode);
1847 rtx temp2 = gen_reg_rtx (DImode);
33b5a38c
TC		 1848 temp1 = simplify_gen_subreg (d->mode, op2, quadmode,
1849 subreg_lowpart_offset (d->mode, quadmode));
9d63f43b 1850 temp1 = simplify_gen_subreg (V2DImode, temp1, d->mode, 0);
33b5a38c
TC		 1851 if (BYTES_BIG_ENDIAN)
1852 emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const0_rtx));
1853 else
1854 emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const1_rtx));
9d63f43b
TC		 1855 op2 = simplify_gen_subreg (d->mode, temp2, GET_MODE (temp2), 0);
1856
1857 /* And recalculate the index. */
1858 lane -= nunits / 4;
1859 }
1860
33b5a38c
TC		 1861 /* Keep to GCC-vector-extension lane indices in the RTL, only nunits / 4
1862 (max nunits in range check) are valid. Which means only 0-1, so we
1863 only need to know the order in a V2mode. */
1864 lane_idx = aarch64_endian_lane_rtx (V2DImode, lane);
1865
fa59c8dc
AC		 1866 if (!target
1867 || !REG_P (target)
1868 || GET_MODE (target) != d->mode)
9d63f43b 1869 target = gen_reg_rtx (d->mode);
9d63f43b
TC		 1870
1871 rtx pat = NULL_RTX;
1872
1873 if (d->lane)
33b5a38c 1874 pat = GEN_FCN (d->icode) (target, op0, op1, op2, lane_idx);
9d63f43b
TC		 1875 else
1876 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
1877
1878 if (!pat)
1879 return NULL_RTX;
1880
1881 emit_insn (pat);
1882 return target;
1883}
1884
89626179
SD		 1885/* Function to expand an expression EXP which calls one of the Transactional
1886 Memory Extension (TME) builtins FCODE with the result going to TARGET. */
1887static rtx
1888aarch64_expand_builtin_tme (int fcode, tree exp, rtx target)
1889{
1890 switch (fcode)
1891 {
1892 case AARCH64_TME_BUILTIN_TSTART:
1893 target = gen_reg_rtx (DImode);
1894 emit_insn (GEN_FCN (CODE_FOR_tstart) (target));
1895 break;
1896
1897 case AARCH64_TME_BUILTIN_TTEST:
1898 target = gen_reg_rtx (DImode);
1899 emit_insn (GEN_FCN (CODE_FOR_ttest) (target));
1900 break;
1901
1902 case AARCH64_TME_BUILTIN_TCOMMIT:
1903 emit_insn (GEN_FCN (CODE_FOR_tcommit) ());
1904 break;
1905
1906 case AARCH64_TME_BUILTIN_TCANCEL:
1907 {
1908 tree arg0 = CALL_EXPR_ARG (exp, 0);
1909 rtx op0 = expand_normal (arg0);
1910 if (CONST_INT_P (op0) && UINTVAL (op0) <= 65536)
1911 emit_insn (GEN_FCN (CODE_FOR_tcancel) (op0));
1912 else
1913 {
1914 error ("%Kargument must be a 16-bit constant immediate", exp);
1915 return const0_rtx;
1916 }
1917 }
1918 break;
1919
1920 default :
1921 gcc_unreachable ();
1922 }
1923 return target;
1924}
1925
c5dc215d
KT		 1926/* Expand a random number builtin EXP with code FCODE, putting the result
1927 int TARGET. If IGNORE is true the return value is ignored. */
1928
1929rtx
1930aarch64_expand_rng_builtin (tree exp, rtx target, int fcode, int ignore)
1931{
1932 rtx pat;
1933 enum insn_code icode;
1934 if (fcode == AARCH64_BUILTIN_RNG_RNDR)
1935 icode = CODE_FOR_aarch64_rndr;
1936 else if (fcode == AARCH64_BUILTIN_RNG_RNDRRS)
1937 icode = CODE_FOR_aarch64_rndrrs;
1938 else
1939 gcc_unreachable ();
1940
1941 rtx rand = gen_reg_rtx (DImode);
1942 pat = GEN_FCN (icode) (rand);
1943 if (!pat)
1944 return NULL_RTX;
1945
1946 tree arg0 = CALL_EXPR_ARG (exp, 0);
1947 rtx res_addr = expand_normal (arg0);
1948 res_addr = convert_memory_address (Pmode, res_addr);
1949 rtx res_mem = gen_rtx_MEM (DImode, res_addr);
1950 emit_insn (pat);
1951 emit_move_insn (res_mem, rand);
1952 /* If the status result is unused don't generate the CSET code. */
1953 if (ignore)
1954 return target;
1955
1956 rtx cc_reg = gen_rtx_REG (CC_Zmode, CC_REGNUM);
f7581eb3 1957 rtx cmp_rtx = gen_rtx_fmt_ee (EQ, SImode, cc_reg, const0_rtx);
c5dc215d
KT		 1958 emit_insn (gen_aarch64_cstoresi (target, cmp_rtx, cc_reg));
1959 return target;
1960}
1961
ef01e6bb
DZ		 1962/* Expand an expression EXP that calls a MEMTAG built-in FCODE
1963 with result going to TARGET. */
1964static rtx
1965aarch64_expand_builtin_memtag (int fcode, tree exp, rtx target)
1966{
1967 if (TARGET_ILP32)
1968 {
1969 error ("Memory Tagging Extension does not support %<-mabi=ilp32%>");
1970 return const0_rtx;
1971 }
1972
1973 rtx pat = NULL;
1974 enum insn_code icode = aarch64_memtag_builtin_data[fcode -
1975 AARCH64_MEMTAG_BUILTIN_START - 1].icode;
1976
1977 rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
1978 machine_mode mode0 = GET_MODE (op0);
1979 op0 = force_reg (mode0 == VOIDmode ? DImode : mode0, op0);
1980 op0 = convert_to_mode (DImode, op0, true);
1981
1982 switch (fcode)
1983 {
1984 case AARCH64_MEMTAG_BUILTIN_IRG:
1985 case AARCH64_MEMTAG_BUILTIN_GMI:
1986 case AARCH64_MEMTAG_BUILTIN_SUBP:
1987 case AARCH64_MEMTAG_BUILTIN_INC_TAG:
1988 {
1989 if (! target
1990 || GET_MODE (target) != DImode
1991 || ! (*insn_data[icode].operand[0].predicate) (target, DImode))
1992 target = gen_reg_rtx (DImode);
1993
1994 if (fcode == AARCH64_MEMTAG_BUILTIN_INC_TAG)
1995 {
1996 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1));
1997
1998 if ((*insn_data[icode].operand[3].predicate) (op1, QImode))
1999 {
2000 pat = GEN_FCN (icode) (target, op0, const0_rtx, op1);
2001 break;
2002 }
2003 error ("%Kargument %d must be a constant immediate "
2004 "in range [0,15]", exp, 2);
2005 return const0_rtx;
2006 }
2007 else
2008 {
2009 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1));
2010 machine_mode mode1 = GET_MODE (op1);
2011 op1 = force_reg (mode1 == VOIDmode ? DImode : mode1, op1);
2012 op1 = convert_to_mode (DImode, op1, true);
2013 pat = GEN_FCN (icode) (target, op0, op1);
2014 }
2015 break;
2016 }
2017 case AARCH64_MEMTAG_BUILTIN_GET_TAG:
2018 target = op0;
2019 pat = GEN_FCN (icode) (target, op0, const0_rtx);
2020 break;
2021 case AARCH64_MEMTAG_BUILTIN_SET_TAG:
2022 pat = GEN_FCN (icode) (op0, op0, const0_rtx);
2023 break;
2024 default:
2025 gcc_unreachable();
2026 }
2027
2028 if (!pat)
2029 return NULL_RTX;
2030
2031 emit_insn (pat);
2032 return target;
2033}
2034
f5e73de0 2035/* Expand an expression EXP as fpsr or fpcr setter (depending on
0d7e5fa6
AC		 2036 UNSPEC) using MODE. */
2037static void
2038aarch64_expand_fpsr_fpcr_setter (int unspec, machine_mode mode, tree exp)
2039{
2040 tree arg = CALL_EXPR_ARG (exp, 0);
2041 rtx op = force_reg (mode, expand_normal (arg));
2042 emit_insn (gen_aarch64_set (unspec, mode, op));
2043}
2044
f5e73de0
AC		 2045/* Expand a fpsr or fpcr getter (depending on UNSPEC) using MODE.
2046 Return the target. */
2047static rtx
2048aarch64_expand_fpsr_fpcr_getter (enum insn_code icode, machine_mode mode,
2049 rtx target)
2050{
2051 expand_operand op;
2052 create_output_operand (&op, target, mode);
2053 expand_insn (icode, 1, &op);
2054 return op.value;
2055}
2056
6d4d616a 2057/* Expand an expression EXP that calls built-in function FCODE,
c5dc215d
KT		 2058 with result going to TARGET if that's convenient. IGNORE is true
2059 if the result of the builtin is ignored. */
342be7f7 2060rtx
c5dc215d
KT		 2061aarch64_general_expand_builtin (unsigned int fcode, tree exp, rtx target,
2062 int ignore)
342be7f7 2063{
aa87aced 2064 int icode;
0d7e5fa6 2065 rtx op0;
aa87aced
KV		 2066 tree arg0;
2067
2068 switch (fcode)
2069 {
2070 case AARCH64_BUILTIN_GET_FPCR:
f5e73de0
AC		 2071 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrsi,
2072 SImode, target);
aa87aced 2073 case AARCH64_BUILTIN_SET_FPCR:
0d7e5fa6
AC		 2074 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR, SImode, exp);
2075 return target;
aa87aced 2076 case AARCH64_BUILTIN_GET_FPSR:
f5e73de0
AC		 2077 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrsi,
2078 SImode, target);
aa87aced 2079 case AARCH64_BUILTIN_SET_FPSR:
0d7e5fa6
AC		 2080 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR, SImode, exp);
2081 return target;
2082 case AARCH64_BUILTIN_GET_FPCR64:
f5e73de0
AC		 2083 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrdi,
2084 DImode, target);
0d7e5fa6
AC		 2085 case AARCH64_BUILTIN_SET_FPCR64:
2086 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR, DImode, exp);
2087 return target;
2088 case AARCH64_BUILTIN_GET_FPSR64:
f5e73de0
AC		 2089 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrdi,
2090 DImode, target);
0d7e5fa6
AC		 2091 case AARCH64_BUILTIN_SET_FPSR64:
2092 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR, DImode, exp);
aa87aced 2093 return target;
312492bd
JW		 2094 case AARCH64_PAUTH_BUILTIN_AUTIA1716:
2095 case AARCH64_PAUTH_BUILTIN_PACIA1716:
8fc16d72
ST		 2096 case AARCH64_PAUTH_BUILTIN_AUTIB1716:
2097 case AARCH64_PAUTH_BUILTIN_PACIB1716:
312492bd
JW		 2098 case AARCH64_PAUTH_BUILTIN_XPACLRI:
2099 arg0 = CALL_EXPR_ARG (exp, 0);
2100 op0 = force_reg (Pmode, expand_normal (arg0));
2101
312492bd
JW		 2102 if (fcode == AARCH64_PAUTH_BUILTIN_XPACLRI)
2103 {
2104 rtx lr = gen_rtx_REG (Pmode, R30_REGNUM);
2105 icode = CODE_FOR_xpaclri;
2106 emit_move_insn (lr, op0);
2107 emit_insn (GEN_FCN (icode) ());
92f0d3d0 2108 return lr;
312492bd
JW		 2109 }
2110 else
2111 {
2112 tree arg1 = CALL_EXPR_ARG (exp, 1);
2113 rtx op1 = force_reg (Pmode, expand_normal (arg1));
8fc16d72
ST		 2114 switch (fcode)
2115 {
2116 case AARCH64_PAUTH_BUILTIN_AUTIA1716:
2117 icode = CODE_FOR_autia1716;
2118 break;
2119 case AARCH64_PAUTH_BUILTIN_AUTIB1716:
2120 icode = CODE_FOR_autib1716;
2121 break;
2122 case AARCH64_PAUTH_BUILTIN_PACIA1716:
2123 icode = CODE_FOR_pacia1716;
2124 break;
2125 case AARCH64_PAUTH_BUILTIN_PACIB1716:
2126 icode = CODE_FOR_pacib1716;
2127 break;
2128 default:
2129 icode = 0;
2130 gcc_unreachable ();
2131 }
312492bd
JW		 2132
2133 rtx x16_reg = gen_rtx_REG (Pmode, R16_REGNUM);
2134 rtx x17_reg = gen_rtx_REG (Pmode, R17_REGNUM);
2135 emit_move_insn (x17_reg, op0);
2136 emit_move_insn (x16_reg, op1);
2137 emit_insn (GEN_FCN (icode) ());
92f0d3d0 2138 return x17_reg;
312492bd
JW		 2139 }
2140
e1d5d19e 2141 case AARCH64_JSCVT:
2c62952f
AC		 2142 {
2143 expand_operand ops[2];
2144 create_output_operand (&ops[0], target, SImode);
2145 op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
2146 create_input_operand (&ops[1], op0, DFmode);
2147 expand_insn (CODE_FOR_aarch64_fjcvtzs, 2, ops);
2148 return ops[0].value;
2149 }
e1d5d19e 2150
9d63f43b
TC		 2151 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V2SF:
2152 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V2SF:
2153 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V2SF:
2154 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V2SF:
2155 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V4HF:
2156 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V4HF:
2157 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V4HF:
2158 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V4HF:
2159 return aarch64_expand_fcmla_builtin (exp, target, fcode);
c5dc215d
KT		 2160 case AARCH64_BUILTIN_RNG_RNDR:
2161 case AARCH64_BUILTIN_RNG_RNDRRS:
2162 return aarch64_expand_rng_builtin (exp, target, fcode, ignore);
aa87aced 2163 }
342be7f7 2164
5d357f26 2165 if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX)
342be7f7 2166 return aarch64_simd_expand_builtin (fcode, exp, target);
5d357f26
KT		 2167 else if (fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX)
2168 return aarch64_crc32_expand_builtin (fcode, exp, target);
342be7f7 2169
a6fc00da
BH		 2170 if (fcode == AARCH64_BUILTIN_RSQRT_DF
2171 || fcode == AARCH64_BUILTIN_RSQRT_SF
2172 || fcode == AARCH64_BUILTIN_RSQRT_V2DF
2173 || fcode == AARCH64_BUILTIN_RSQRT_V2SF
2174 || fcode == AARCH64_BUILTIN_RSQRT_V4SF)
2175 return aarch64_expand_builtin_rsqrt (fcode, exp, target);
2176
89626179
SD		 2177 if (fcode == AARCH64_TME_BUILTIN_TSTART
2178 || fcode == AARCH64_TME_BUILTIN_TCOMMIT
2179 || fcode == AARCH64_TME_BUILTIN_TTEST
2180 || fcode == AARCH64_TME_BUILTIN_TCANCEL)
2181 return aarch64_expand_builtin_tme (fcode, exp, target);
2182
ef01e6bb
DZ		 2183 if (fcode >= AARCH64_MEMTAG_BUILTIN_START
2184 && fcode <= AARCH64_MEMTAG_BUILTIN_END)
2185 return aarch64_expand_builtin_memtag (fcode, exp, target);
2186
d5a29419 2187 gcc_unreachable ();
342be7f7 2188}
42fc9a7f
JG		 2189
2190tree
10766209
RS		 2191aarch64_builtin_vectorized_function (unsigned int fn, tree type_out,
2192 tree type_in)
42fc9a7f 2193{
ef4bddc2 2194 machine_mode in_mode, out_mode;
42fc9a7f
JG		 2195
2196 if (TREE_CODE (type_out) != VECTOR_TYPE
2197 || TREE_CODE (type_in) != VECTOR_TYPE)
2198 return NULL_TREE;
2199
7cee9637
RS		 2200 out_mode = TYPE_MODE (type_out);
2201 in_mode = TYPE_MODE (type_in);
42fc9a7f
JG		 2202
2203#undef AARCH64_CHECK_BUILTIN_MODE
2204#define AARCH64_CHECK_BUILTIN_MODE(C, N) 1
2205#define AARCH64_FIND_FRINT_VARIANT(N) \
2206 (AARCH64_CHECK_BUILTIN_MODE (2, D) \
e993fea1 2207 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2df] \
42fc9a7f 2208 : (AARCH64_CHECK_BUILTIN_MODE (4, S) \
e993fea1 2209 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v4sf] \
42fc9a7f 2210 : (AARCH64_CHECK_BUILTIN_MODE (2, S) \
e993fea1 2211 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2sf] \
42fc9a7f 2212 : NULL_TREE)))
10766209 2213 switch (fn)
42fc9a7f 2214 {
42fc9a7f
JG		 2215#undef AARCH64_CHECK_BUILTIN_MODE
2216#define AARCH64_CHECK_BUILTIN_MODE(C, N) \
7cee9637 2217 (out_mode == V##C##N##Fmode && in_mode == V##C##N##Fmode)
10766209
RS		 2218 CASE_CFN_FLOOR:
2219 return AARCH64_FIND_FRINT_VARIANT (floor);
2220 CASE_CFN_CEIL:
2221 return AARCH64_FIND_FRINT_VARIANT (ceil);
2222 CASE_CFN_TRUNC:
2223 return AARCH64_FIND_FRINT_VARIANT (btrunc);
2224 CASE_CFN_ROUND:
2225 return AARCH64_FIND_FRINT_VARIANT (round);
2226 CASE_CFN_NEARBYINT:
2227 return AARCH64_FIND_FRINT_VARIANT (nearbyint);
2228 CASE_CFN_SQRT:
2229 return AARCH64_FIND_FRINT_VARIANT (sqrt);
42fc9a7f 2230#undef AARCH64_CHECK_BUILTIN_MODE
b5574232 2231#define AARCH64_CHECK_BUILTIN_MODE(C, N) \
7cee9637 2232 (out_mode == V##C##SImode && in_mode == V##C##N##Imode)
10766209
RS		 2233 CASE_CFN_CLZ:
2234 {
2235 if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2236 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_clzv4si];
2237 return NULL_TREE;
2238 }
2239 CASE_CFN_CTZ:
2240 {
2241 if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2242 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv2si];
2243 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2244 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv4si];
2245 return NULL_TREE;
2246 }
b5574232 2247#undef AARCH64_CHECK_BUILTIN_MODE
42fc9a7f 2248#define AARCH64_CHECK_BUILTIN_MODE(C, N) \
7cee9637 2249 (out_mode == V##C##N##Imode && in_mode == V##C##N##Fmode)
10766209
RS		 2250 CASE_CFN_IFLOOR:
2251 CASE_CFN_LFLOOR:
2252 CASE_CFN_LLFLOOR:
2253 {
2254 enum aarch64_builtins builtin;
2255 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2256 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2dfv2di;
2257 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2258 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv4sfv4si;
2259 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2260 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2sfv2si;
2261 else
2262 return NULL_TREE;
2263
2264 return aarch64_builtin_decls[builtin];
2265 }
2266 CASE_CFN_ICEIL:
2267 CASE_CFN_LCEIL:
2268 CASE_CFN_LLCEIL:
2269 {
2270 enum aarch64_builtins builtin;
2271 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2272 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2dfv2di;
2273 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2274 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv4sfv4si;
2275 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2276 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2sfv2si;
2277 else
2278 return NULL_TREE;
2279
2280 return aarch64_builtin_decls[builtin];
2281 }
2282 CASE_CFN_IROUND:
2283 CASE_CFN_LROUND:
2284 CASE_CFN_LLROUND:
2285 {
2286 enum aarch64_builtins builtin;
2287 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2288 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2dfv2di;
2289 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2290 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv4sfv4si;
2291 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2292 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2sfv2si;
2293 else
2294 return NULL_TREE;
2295
2296 return aarch64_builtin_decls[builtin];
2297 }
10766209
RS		 2298 default:
2299 return NULL_TREE;
42fc9a7f
JG		 2300 }
2301
2302 return NULL_TREE;
2303}
0ac198d3 2304
a6fc00da
BH		 2305/* Return builtin for reciprocal square root. */
2306
2307tree
6d4d616a 2308aarch64_general_builtin_rsqrt (unsigned int fn)
a6fc00da 2309{
ee62a5a6
RS		 2310 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df)
2311 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2DF];
2312 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf)
2313 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2SF];
2314 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf)
2315 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V4SF];
a6fc00da
BH		 2316 return NULL_TREE;
2317}
2318
0ac198d3 2319#undef VAR1
bf592b2f 2320#define VAR1(T, N, MAP, FLAG, A) \
e993fea1 2321 case AARCH64_SIMD_BUILTIN_##T##_##N##A:
0ac198d3 2322
6d4d616a
RS		 2323/* Try to fold a call to the built-in function with subcode FCODE. The
2324 function is passed the N_ARGS arguments in ARGS and it returns a value
2325 of type TYPE. Return the new expression on success and NULL_TREE on
2326 failure. */
9697e620 2327tree
6d4d616a
RS		 2328aarch64_general_fold_builtin (unsigned int fcode, tree type,
2329 unsigned int n_args ATTRIBUTE_UNUSED, tree *args)
9697e620 2330{
9697e620
JG		 2331 switch (fcode)
2332 {
bf592b2f 2333 BUILTIN_VDQF (UNOP, abs, 2, ALL)
9697e620 2334 return fold_build1 (ABS_EXPR, type, args[0]);
bf592b2f 2335 VAR1 (UNOP, floatv2si, 2, ALL, v2sf)
2336 VAR1 (UNOP, floatv4si, 2, ALL, v4sf)
2337 VAR1 (UNOP, floatv2di, 2, ALL, v2df)
1709ff9b 2338 return fold_build1 (FLOAT_EXPR, type, args[0]);
9697e620
JG		 2339 default:
2340 break;
2341 }
2342
2343 return NULL_TREE;
2344}
2345
6d4d616a
RS		 2346/* Try to fold STMT, given that it's a call to the built-in function with
2347 subcode FCODE. Return the new statement on success and null on
2348 failure. */
2349gimple *
2350aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt)
0ac198d3 2351{
355fe088 2352 gimple *new_stmt = NULL;
6d4d616a
RS		 2353 unsigned nargs = gimple_call_num_args (stmt);
2354 tree *args = (nargs > 0
2355 ? gimple_call_arg_ptr (stmt, 0)
2356 : &error_mark_node);
2357
2358 /* We use gimple's IFN_REDUC_(PLUS|MIN|MAX)s for float, signed int
2359 and unsigned int; it will distinguish according to the types of
2360 the arguments to the __builtin. */
2361 switch (fcode)
0ac198d3 2362 {
bf592b2f 2363 BUILTIN_VALL (UNOP, reduc_plus_scal_, 10, ALL)
6d4d616a
RS		 2364 new_stmt = gimple_build_call_internal (IFN_REDUC_PLUS,
2365 1, args[0]);
2366 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2367 break;
bf592b2f 2368 BUILTIN_VDQIF (UNOP, reduc_smax_scal_, 10, ALL)
2369 BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10, ALL)
6d4d616a
RS		 2370 new_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
2371 1, args[0]);
2372 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2373 break;
bf592b2f 2374 BUILTIN_VDQIF (UNOP, reduc_smin_scal_, 10, ALL)
2375 BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10, ALL)
6d4d616a
RS		 2376 new_stmt = gimple_build_call_internal (IFN_REDUC_MIN,
2377 1, args[0]);
2378 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2379 break;
bf592b2f 2380 BUILTIN_GPF (BINOP, fmulx, 0, ALL)
0ac198d3 2381 {
6d4d616a
RS		 2382 gcc_assert (nargs == 2);
2383 bool a0_cst_p = TREE_CODE (args[0]) == REAL_CST;
2384 bool a1_cst_p = TREE_CODE (args[1]) == REAL_CST;
2385 if (a0_cst_p || a1_cst_p)
0ac198d3 2386 {
6d4d616a 2387 if (a0_cst_p && a1_cst_p)
546e500c 2388 {
6d4d616a
RS		 2389 tree t0 = TREE_TYPE (args[0]);
2390 real_value a0 = (TREE_REAL_CST (args[0]));
2391 real_value a1 = (TREE_REAL_CST (args[1]));
2392 if (real_equal (&a1, &dconst0))
2393 std::swap (a0, a1);
2394 /* According to real_equal (), +0 equals -0. */
2395 if (real_equal (&a0, &dconst0) && real_isinf (&a1))
546e500c 2396 {
6d4d616a
RS		 2397 real_value res = dconst2;
2398 res.sign = a0.sign ^ a1.sign;
2399 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2400 REAL_CST,
2401 build_real (t0, res));
546e500c 2402 }
6d4d616a
RS		 2403 else
2404 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2405 MULT_EXPR,
2406 args[0], args[1]);
546e500c 2407 }
6d4d616a
RS		 2408 else /* a0_cst_p ^ a1_cst_p. */
2409 {
2410 real_value const_part = a0_cst_p
2411 ? TREE_REAL_CST (args[0]) : TREE_REAL_CST (args[1]);
2412 if (!real_equal (&const_part, &dconst0)
2413 && !real_isinf (&const_part))
2414 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2415 MULT_EXPR, args[0],
2416 args[1]);
2417 }
2418 }
2419 if (new_stmt)
2420 {
2421 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
2422 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
0ac198d3 2423 }
6d4d616a 2424 break;
0ac198d3 2425 }
6d4d616a
RS		 2426 default:
2427 break;
0ac198d3 2428 }
6d4d616a 2429 return new_stmt;
0ac198d3
JG		 2430}
2431
aa87aced
KV		 2432void
2433aarch64_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
2434{
2435 const unsigned AARCH64_FE_INVALID = 1;
2436 const unsigned AARCH64_FE_DIVBYZERO = 2;
2437 const unsigned AARCH64_FE_OVERFLOW = 4;
2438 const unsigned AARCH64_FE_UNDERFLOW = 8;
2439 const unsigned AARCH64_FE_INEXACT = 16;
2440 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT = (AARCH64_FE_INVALID
2441 | AARCH64_FE_DIVBYZERO
2442 | AARCH64_FE_OVERFLOW
2443 | AARCH64_FE_UNDERFLOW
2444 | AARCH64_FE_INEXACT);
2445 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT = 8;
2446 tree fenv_cr, fenv_sr, get_fpcr, set_fpcr, mask_cr, mask_sr;
2447 tree ld_fenv_cr, ld_fenv_sr, masked_fenv_cr, masked_fenv_sr, hold_fnclex_cr;
2448 tree hold_fnclex_sr, new_fenv_var, reload_fenv, restore_fnenv, get_fpsr, set_fpsr;
2449 tree update_call, atomic_feraiseexcept, hold_fnclex, masked_fenv, ld_fenv;
2450
2451 /* Generate the equivalence of :
2452 unsigned int fenv_cr;
2453 fenv_cr = __builtin_aarch64_get_fpcr ();
2454
2455 unsigned int fenv_sr;
2456 fenv_sr = __builtin_aarch64_get_fpsr ();
2457
2458 Now set all exceptions to non-stop
2459 unsigned int mask_cr
2460 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT);
2461 unsigned int masked_cr;
2462 masked_cr = fenv_cr & mask_cr;
2463
2464 And clear all exception flags
2465 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT;
2466 unsigned int masked_cr;
2467 masked_sr = fenv_sr & mask_sr;
2468
2469 __builtin_aarch64_set_cr (masked_cr);
2470 __builtin_aarch64_set_sr (masked_sr); */
2471
09ba9ef7
RR		 2472 fenv_cr = create_tmp_var_raw (unsigned_type_node);
2473 fenv_sr = create_tmp_var_raw (unsigned_type_node);
aa87aced
KV		 2474
2475 get_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR];
2476 set_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR];
2477 get_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR];
2478 set_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR];
2479
2480 mask_cr = build_int_cst (unsigned_type_node,
2481 ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT));
2482 mask_sr = build_int_cst (unsigned_type_node,
2483 ~(AARCH64_FE_ALL_EXCEPT));
2484
d81bc2af
HZ		 2485 ld_fenv_cr = build4 (TARGET_EXPR, unsigned_type_node,
2486 fenv_cr, build_call_expr (get_fpcr, 0),
2487 NULL_TREE, NULL_TREE);
2488 ld_fenv_sr = build4 (TARGET_EXPR, unsigned_type_node,
2489 fenv_sr, build_call_expr (get_fpsr, 0),
2490 NULL_TREE, NULL_TREE);
aa87aced
KV		 2491
2492 masked_fenv_cr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_cr, mask_cr);
2493 masked_fenv_sr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_sr, mask_sr);
2494
2495 hold_fnclex_cr = build_call_expr (set_fpcr, 1, masked_fenv_cr);
2496 hold_fnclex_sr = build_call_expr (set_fpsr, 1, masked_fenv_sr);
2497
2498 hold_fnclex = build2 (COMPOUND_EXPR, void_type_node, hold_fnclex_cr,
2499 hold_fnclex_sr);
2500 masked_fenv = build2 (COMPOUND_EXPR, void_type_node, masked_fenv_cr,
2501 masked_fenv_sr);
2502 ld_fenv = build2 (COMPOUND_EXPR, void_type_node, ld_fenv_cr, ld_fenv_sr);
2503
2504 *hold = build2 (COMPOUND_EXPR, void_type_node,
2505 build2 (COMPOUND_EXPR, void_type_node, masked_fenv, ld_fenv),
2506 hold_fnclex);
2507
2508 /* Store the value of masked_fenv to clear the exceptions:
2509 __builtin_aarch64_set_fpsr (masked_fenv_sr); */
2510
2511 *clear = build_call_expr (set_fpsr, 1, masked_fenv_sr);
2512
2513 /* Generate the equivalent of :
2514 unsigned int new_fenv_var;
2515 new_fenv_var = __builtin_aarch64_get_fpsr ();
2516
2517 __builtin_aarch64_set_fpsr (fenv_sr);
2518
2519 __atomic_feraiseexcept (new_fenv_var); */
2520
09ba9ef7 2521 new_fenv_var = create_tmp_var_raw (unsigned_type_node);
d81bc2af
HZ		 2522 reload_fenv = build4 (TARGET_EXPR, unsigned_type_node,
2523 new_fenv_var, build_call_expr (get_fpsr, 0),
2524 NULL_TREE, NULL_TREE);
aa87aced
KV		 2525 restore_fnenv = build_call_expr (set_fpsr, 1, fenv_sr);
2526 atomic_feraiseexcept = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
2527 update_call = build_call_expr (atomic_feraiseexcept, 1,
2528 fold_convert (integer_type_node, new_fenv_var));
2529 *update = build2 (COMPOUND_EXPR, void_type_node,
2530 build2 (COMPOUND_EXPR, void_type_node,
2531 reload_fenv, restore_fnenv), update_call);
2532}
2533
ef01e6bb
DZ		 2534/* Resolve overloaded MEMTAG build-in functions. */
2535#define AARCH64_BUILTIN_SUBCODE(F) \
2536 (DECL_MD_FUNCTION_CODE (F) >> AARCH64_BUILTIN_SHIFT)
2537
2538static tree
2539aarch64_resolve_overloaded_memtag (location_t loc,
2540 tree fndecl, void *pass_params)
2541{
2542 vec<tree, va_gc> *params = static_cast<vec<tree, va_gc> *> (pass_params);
2543 unsigned param_num = params ? params->length() : 0;
2544 unsigned int fcode = AARCH64_BUILTIN_SUBCODE (fndecl);
2545 tree inittype = aarch64_memtag_builtin_data[
2546 fcode - AARCH64_MEMTAG_BUILTIN_START - 1].ftype;
2547 unsigned arg_num = list_length (TYPE_ARG_TYPES (inittype)) - 1;
2548
2549 if (param_num != arg_num)
2550 {
2551 TREE_TYPE (fndecl) = inittype;
2552 return NULL_TREE;
2553 }
2554 tree retype = NULL;
2555
2556 if (fcode == AARCH64_MEMTAG_BUILTIN_SUBP)
2557 {
2558 tree t0 = TREE_TYPE ((*params)[0]);
2559 tree t1 = TREE_TYPE ((*params)[1]);
2560
2561 if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE)
2562 t0 = ptr_type_node;
2563 if (t1 == error_mark_node || TREE_CODE (t1) != POINTER_TYPE)
2564 t1 = ptr_type_node;
2565
2566 if (TYPE_MODE (t0) != DImode)
2567 warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit",
2568 (int)tree_to_shwi (DECL_SIZE ((*params)[0])));
2569
2570 if (TYPE_MODE (t1) != DImode)
2571 warning_at (loc, 1, "expected 64-bit address but argument 2 is %d-bit",
2572 (int)tree_to_shwi (DECL_SIZE ((*params)[1])));
2573
2574 retype = build_function_type_list (ptrdiff_type_node, t0, t1, NULL);
2575 }
2576 else
2577 {
2578 tree t0 = TREE_TYPE ((*params)[0]);
2579
2580 if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE)
2581 {
2582 TREE_TYPE (fndecl) = inittype;
2583 return NULL_TREE;
2584 }
2585
2586 if (TYPE_MODE (t0) != DImode)
2587 warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit",
2588 (int)tree_to_shwi (DECL_SIZE ((*params)[0])));
2589
2590 switch (fcode)
2591 {
2592 case AARCH64_MEMTAG_BUILTIN_IRG:
2593 retype = build_function_type_list (t0, t0, uint64_type_node, NULL);
2594 break;
2595 case AARCH64_MEMTAG_BUILTIN_GMI:
2596 retype = build_function_type_list (uint64_type_node, t0,
2597 uint64_type_node, NULL);
2598 break;
2599 case AARCH64_MEMTAG_BUILTIN_INC_TAG:
2600 retype = build_function_type_list (t0, t0, unsigned_type_node, NULL);
2601 break;
2602 case AARCH64_MEMTAG_BUILTIN_SET_TAG:
2603 retype = build_function_type_list (void_type_node, t0, NULL);
2604 break;
2605 case AARCH64_MEMTAG_BUILTIN_GET_TAG:
2606 retype = build_function_type_list (t0, t0, NULL);
2607 break;
2608 default:
2609 return NULL_TREE;
2610 }
2611 }
2612
2613 if (!retype || retype == error_mark_node)
2614 TREE_TYPE (fndecl) = inittype;
2615 else
2616 TREE_TYPE (fndecl) = retype;
2617
2618 return NULL_TREE;
2619}
2620
2621/* Called at aarch64_resolve_overloaded_builtin in aarch64-c.c. */
2622tree
2623aarch64_resolve_overloaded_builtin_general (location_t loc, tree function,
2624 void *pass_params)
2625{
2626 unsigned int fcode = AARCH64_BUILTIN_SUBCODE (function);
2627
2628 if (fcode >= AARCH64_MEMTAG_BUILTIN_START
2629 && fcode <= AARCH64_MEMTAG_BUILTIN_END)
2630 return aarch64_resolve_overloaded_memtag(loc, function, pass_params);
2631
2632 return NULL_TREE;
2633}
aa87aced 2634
42fc9a7f
JG		 2635#undef AARCH64_CHECK_BUILTIN_MODE
2636#undef AARCH64_FIND_FRINT_VARIANT
0ddec79f
JG		 2637#undef CF0
2638#undef CF1
2639#undef CF2
2640#undef CF3
2641#undef CF4
2642#undef CF10
2643#undef VAR1
2644#undef VAR2
2645#undef VAR3
2646#undef VAR4
2647#undef VAR5
2648#undef VAR6
2649#undef VAR7
2650#undef VAR8
2651#undef VAR9
2652#undef VAR10
2653#undef VAR11
2654
3c03d39d 2655#include "gt-aarch64-builtins.h"
GNU Compiler Collection
This page took 2.993333 seconds and 5 git commands to generate.