]> gcc.gnu.org Git - gcc.git/blob - gcc/internal-fn.cc
ipa/105166 - avoid modref queries with mismatching types
[gcc.git] / gcc / internal-fn.cc
1 /* Internal functions.
2 Copyright (C) 2011-2022 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "target.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "predict.h"
29 #include "stringpool.h"
30 #include "tree-vrp.h"
31 #include "tree-ssanames.h"
32 #include "expmed.h"
33 #include "memmodel.h"
34 #include "optabs.h"
35 #include "emit-rtl.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
40 #include "dojump.h"
41 #include "expr.h"
42 #include "stringpool.h"
43 #include "attribs.h"
44 #include "asan.h"
45 #include "ubsan.h"
46 #include "recog.h"
47 #include "builtins.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
52 #include "explow.h"
53 #include "rtl-iter.h"
54 #include "gimple-range.h"
55
56 /* For lang_hooks.types.type_for_mode. */
57 #include "langhooks.h"
58
59 /* The names of each internal function, indexed by function number. */
60 const char *const internal_fn_name_array[] = {
61 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
62 #include "internal-fn.def"
63 "<invalid-fn>"
64 };
65
66 /* The ECF_* flags of each internal function, indexed by function number. */
67 const int internal_fn_flags_array[] = {
68 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
69 #include "internal-fn.def"
70 0
71 };
72
73 /* Return the internal function called NAME, or IFN_LAST if there's
74 no such function. */
75
76 internal_fn
77 lookup_internal_fn (const char *name)
78 {
79 typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
80 static name_to_fn_map_type *name_to_fn_map;
81
82 if (!name_to_fn_map)
83 {
84 name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
85 for (unsigned int i = 0; i < IFN_LAST; ++i)
86 name_to_fn_map->put (internal_fn_name (internal_fn (i)),
87 internal_fn (i));
88 }
89 internal_fn *entry = name_to_fn_map->get (name);
90 return entry ? *entry : IFN_LAST;
91 }
92
93 /* Fnspec of each internal function, indexed by function number. */
94 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
95
96 void
97 init_internal_fns ()
98 {
99 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
100 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
101 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
102 #include "internal-fn.def"
103 internal_fn_fnspec_array[IFN_LAST] = 0;
104 }
105
106 /* Create static initializers for the information returned by
107 direct_internal_fn. */
108 #define not_direct { -2, -2, false }
109 #define mask_load_direct { -1, 2, false }
110 #define load_lanes_direct { -1, -1, false }
111 #define mask_load_lanes_direct { -1, -1, false }
112 #define gather_load_direct { 3, 1, false }
113 #define len_load_direct { -1, -1, false }
114 #define mask_store_direct { 3, 2, false }
115 #define store_lanes_direct { 0, 0, false }
116 #define mask_store_lanes_direct { 0, 0, false }
117 #define vec_cond_mask_direct { 1, 0, false }
118 #define vec_cond_direct { 2, 0, false }
119 #define scatter_store_direct { 3, 1, false }
120 #define len_store_direct { 3, 3, false }
121 #define vec_set_direct { 3, 3, false }
122 #define unary_direct { 0, 0, true }
123 #define binary_direct { 0, 0, true }
124 #define ternary_direct { 0, 0, true }
125 #define cond_unary_direct { 1, 1, true }
126 #define cond_binary_direct { 1, 1, true }
127 #define cond_ternary_direct { 1, 1, true }
128 #define while_direct { 0, 2, false }
129 #define fold_extract_direct { 2, 2, false }
130 #define fold_left_direct { 1, 1, false }
131 #define mask_fold_left_direct { 1, 1, false }
132 #define check_ptrs_direct { 0, 0, false }
133
134 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
135 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
136 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
137 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
138 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
139 #include "internal-fn.def"
140 not_direct
141 };
142
143 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
144 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
145
146 static enum insn_code
147 get_multi_vector_move (tree array_type, convert_optab optab)
148 {
149 machine_mode imode;
150 machine_mode vmode;
151
152 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
153 imode = TYPE_MODE (array_type);
154 vmode = TYPE_MODE (TREE_TYPE (array_type));
155
156 return convert_optab_handler (optab, imode, vmode);
157 }
158
159 /* Expand LOAD_LANES call STMT using optab OPTAB. */
160
161 static void
162 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
163 {
164 class expand_operand ops[2];
165 tree type, lhs, rhs;
166 rtx target, mem;
167
168 lhs = gimple_call_lhs (stmt);
169 rhs = gimple_call_arg (stmt, 0);
170 type = TREE_TYPE (lhs);
171
172 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
173 mem = expand_normal (rhs);
174
175 gcc_assert (MEM_P (mem));
176 PUT_MODE (mem, TYPE_MODE (type));
177
178 create_output_operand (&ops[0], target, TYPE_MODE (type));
179 create_fixed_operand (&ops[1], mem);
180 expand_insn (get_multi_vector_move (type, optab), 2, ops);
181 if (!rtx_equal_p (target, ops[0].value))
182 emit_move_insn (target, ops[0].value);
183 }
184
185 /* Expand STORE_LANES call STMT using optab OPTAB. */
186
187 static void
188 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
189 {
190 class expand_operand ops[2];
191 tree type, lhs, rhs;
192 rtx target, reg;
193
194 lhs = gimple_call_lhs (stmt);
195 rhs = gimple_call_arg (stmt, 0);
196 type = TREE_TYPE (rhs);
197
198 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
199 reg = expand_normal (rhs);
200
201 gcc_assert (MEM_P (target));
202 PUT_MODE (target, TYPE_MODE (type));
203
204 create_fixed_operand (&ops[0], target);
205 create_input_operand (&ops[1], reg, TYPE_MODE (type));
206 expand_insn (get_multi_vector_move (type, optab), 2, ops);
207 }
208
209 static void
210 expand_ANNOTATE (internal_fn, gcall *)
211 {
212 gcc_unreachable ();
213 }
214
215 /* This should get expanded in omp_device_lower pass. */
216
217 static void
218 expand_GOMP_USE_SIMT (internal_fn, gcall *)
219 {
220 gcc_unreachable ();
221 }
222
223 /* This should get expanded in omp_device_lower pass. */
224
225 static void
226 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
227 {
228 gcc_unreachable ();
229 }
230
231 /* Allocate per-lane storage and begin non-uniform execution region. */
232
233 static void
234 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
235 {
236 rtx target;
237 tree lhs = gimple_call_lhs (stmt);
238 if (lhs)
239 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
240 else
241 target = gen_reg_rtx (Pmode);
242 rtx size = expand_normal (gimple_call_arg (stmt, 0));
243 rtx align = expand_normal (gimple_call_arg (stmt, 1));
244 class expand_operand ops[3];
245 create_output_operand (&ops[0], target, Pmode);
246 create_input_operand (&ops[1], size, Pmode);
247 create_input_operand (&ops[2], align, Pmode);
248 gcc_assert (targetm.have_omp_simt_enter ());
249 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
250 if (!rtx_equal_p (target, ops[0].value))
251 emit_move_insn (target, ops[0].value);
252 }
253
254 /* Deallocate per-lane storage and leave non-uniform execution region. */
255
256 static void
257 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
258 {
259 gcc_checking_assert (!gimple_call_lhs (stmt));
260 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
261 class expand_operand ops[1];
262 create_input_operand (&ops[0], arg, Pmode);
263 gcc_assert (targetm.have_omp_simt_exit ());
264 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
265 }
266
267 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
268 without SIMT execution this should be expanded in omp_device_lower pass. */
269
270 static void
271 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
272 {
273 tree lhs = gimple_call_lhs (stmt);
274 if (!lhs)
275 return;
276
277 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
278 gcc_assert (targetm.have_omp_simt_lane ());
279 emit_insn (targetm.gen_omp_simt_lane (target));
280 }
281
282 /* This should get expanded in omp_device_lower pass. */
283
284 static void
285 expand_GOMP_SIMT_VF (internal_fn, gcall *)
286 {
287 gcc_unreachable ();
288 }
289
290 /* Lane index of the first SIMT lane that supplies a non-zero argument.
291 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
292 lane that executed the last iteration for handling OpenMP lastprivate. */
293
294 static void
295 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
296 {
297 tree lhs = gimple_call_lhs (stmt);
298 if (!lhs)
299 return;
300
301 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
302 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
303 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
304 class expand_operand ops[2];
305 create_output_operand (&ops[0], target, mode);
306 create_input_operand (&ops[1], cond, mode);
307 gcc_assert (targetm.have_omp_simt_last_lane ());
308 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
309 if (!rtx_equal_p (target, ops[0].value))
310 emit_move_insn (target, ops[0].value);
311 }
312
313 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
314
315 static void
316 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
317 {
318 tree lhs = gimple_call_lhs (stmt);
319 if (!lhs)
320 return;
321
322 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
323 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
324 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
325 class expand_operand ops[2];
326 create_output_operand (&ops[0], target, mode);
327 create_input_operand (&ops[1], ctr, mode);
328 gcc_assert (targetm.have_omp_simt_ordered ());
329 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
330 if (!rtx_equal_p (target, ops[0].value))
331 emit_move_insn (target, ops[0].value);
332 }
333
334 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
335 any lane supplies a non-zero argument. */
336
337 static void
338 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
339 {
340 tree lhs = gimple_call_lhs (stmt);
341 if (!lhs)
342 return;
343
344 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
345 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
346 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
347 class expand_operand ops[2];
348 create_output_operand (&ops[0], target, mode);
349 create_input_operand (&ops[1], cond, mode);
350 gcc_assert (targetm.have_omp_simt_vote_any ());
351 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
352 if (!rtx_equal_p (target, ops[0].value))
353 emit_move_insn (target, ops[0].value);
354 }
355
356 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
357 is destination lane index XOR given offset. */
358
359 static void
360 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
361 {
362 tree lhs = gimple_call_lhs (stmt);
363 if (!lhs)
364 return;
365
366 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
367 rtx src = expand_normal (gimple_call_arg (stmt, 0));
368 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
369 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
370 class expand_operand ops[3];
371 create_output_operand (&ops[0], target, mode);
372 create_input_operand (&ops[1], src, mode);
373 create_input_operand (&ops[2], idx, SImode);
374 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
375 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
376 if (!rtx_equal_p (target, ops[0].value))
377 emit_move_insn (target, ops[0].value);
378 }
379
380 /* Exchange between SIMT lanes according to given source lane index. */
381
382 static void
383 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
384 {
385 tree lhs = gimple_call_lhs (stmt);
386 if (!lhs)
387 return;
388
389 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
390 rtx src = expand_normal (gimple_call_arg (stmt, 0));
391 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
392 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
393 class expand_operand ops[3];
394 create_output_operand (&ops[0], target, mode);
395 create_input_operand (&ops[1], src, mode);
396 create_input_operand (&ops[2], idx, SImode);
397 gcc_assert (targetm.have_omp_simt_xchg_idx ());
398 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
399 if (!rtx_equal_p (target, ops[0].value))
400 emit_move_insn (target, ops[0].value);
401 }
402
403 /* This should get expanded in adjust_simduid_builtins. */
404
405 static void
406 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
407 {
408 gcc_unreachable ();
409 }
410
411 /* This should get expanded in adjust_simduid_builtins. */
412
413 static void
414 expand_GOMP_SIMD_VF (internal_fn, gcall *)
415 {
416 gcc_unreachable ();
417 }
418
419 /* This should get expanded in adjust_simduid_builtins. */
420
421 static void
422 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
423 {
424 gcc_unreachable ();
425 }
426
427 /* This should get expanded in adjust_simduid_builtins. */
428
429 static void
430 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
431 {
432 gcc_unreachable ();
433 }
434
435 /* This should get expanded in adjust_simduid_builtins. */
436
437 static void
438 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
439 {
440 gcc_unreachable ();
441 }
442
443 /* This should get expanded in the sanopt pass. */
444
445 static void
446 expand_UBSAN_NULL (internal_fn, gcall *)
447 {
448 gcc_unreachable ();
449 }
450
451 /* This should get expanded in the sanopt pass. */
452
453 static void
454 expand_UBSAN_BOUNDS (internal_fn, gcall *)
455 {
456 gcc_unreachable ();
457 }
458
459 /* This should get expanded in the sanopt pass. */
460
461 static void
462 expand_UBSAN_VPTR (internal_fn, gcall *)
463 {
464 gcc_unreachable ();
465 }
466
467 /* This should get expanded in the sanopt pass. */
468
469 static void
470 expand_UBSAN_PTR (internal_fn, gcall *)
471 {
472 gcc_unreachable ();
473 }
474
475 /* This should get expanded in the sanopt pass. */
476
477 static void
478 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
479 {
480 gcc_unreachable ();
481 }
482
483 /* This should get expanded in the sanopt pass. */
484
485 static void
486 expand_HWASAN_CHECK (internal_fn, gcall *)
487 {
488 gcc_unreachable ();
489 }
490
491 /* For hwasan stack tagging:
492 Clear tags on the dynamically allocated space.
493 For use after an object dynamically allocated on the stack goes out of
494 scope. */
495 static void
496 expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
497 {
498 gcc_assert (Pmode == ptr_mode);
499 tree restored_position = gimple_call_arg (gc, 0);
500 rtx restored_rtx = expand_expr (restored_position, NULL_RTX, VOIDmode,
501 EXPAND_NORMAL);
502 rtx func = init_one_libfunc ("__hwasan_tag_memory");
503 rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
504 stack_pointer_rtx, NULL_RTX, 0,
505 OPTAB_WIDEN);
506 emit_library_call_value (func, NULL_RTX, LCT_NORMAL, VOIDmode,
507 virtual_stack_dynamic_rtx, Pmode,
508 HWASAN_STACK_BACKGROUND, QImode,
509 off, Pmode);
510 }
511
512 /* For hwasan stack tagging:
513 Return a tag to be used for a dynamic allocation. */
514 static void
515 expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
516 {
517 tree tag = gimple_call_lhs (gc);
518 rtx target = expand_expr (tag, NULL_RTX, VOIDmode, EXPAND_NORMAL);
519 machine_mode mode = GET_MODE (target);
520 gcc_assert (mode == QImode);
521
522 rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
523 gcc_assert (base_tag);
524 rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
525 rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
526 target, /* unsignedp = */1,
527 OPTAB_WIDEN);
528 chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
529
530 /* Really need to put the tag into the `target` RTX. */
531 if (chosen_tag != target)
532 {
533 rtx temp = chosen_tag;
534 gcc_assert (GET_MODE (chosen_tag) == mode);
535 emit_move_insn (target, temp);
536 }
537
538 hwasan_increment_frame_tag ();
539 }
540
541 /* For hwasan stack tagging:
542 Tag a region of space in the shadow stack according to the base pointer of
543 an object on the stack. N.b. the length provided in the internal call is
544 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
545 static void
546 expand_HWASAN_MARK (internal_fn, gcall *gc)
547 {
548 gcc_assert (ptr_mode == Pmode);
549 HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gc, 0));
550 bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
551
552 tree base = gimple_call_arg (gc, 1);
553 gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
554 rtx base_rtx = expand_normal (base);
555
556 rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
557 : targetm.memtag.extract_tag (base_rtx, NULL_RTX);
558 rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
559
560 tree len = gimple_call_arg (gc, 2);
561 rtx r_len = expand_normal (len);
562
563 rtx func = init_one_libfunc ("__hwasan_tag_memory");
564 emit_library_call (func, LCT_NORMAL, VOIDmode, address, Pmode,
565 tag, QImode, r_len, Pmode);
566 }
567
568 /* For hwasan stack tagging:
569 Store a tag into a pointer. */
570 static void
571 expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
572 {
573 gcc_assert (ptr_mode == Pmode);
574 tree g_target = gimple_call_lhs (gc);
575 tree g_ptr = gimple_call_arg (gc, 0);
576 tree g_tag = gimple_call_arg (gc, 1);
577
578 rtx ptr = expand_normal (g_ptr);
579 rtx tag = expand_expr (g_tag, NULL_RTX, QImode, EXPAND_NORMAL);
580 rtx target = expand_normal (g_target);
581
582 rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
583 rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
584 if (tagged_value != target)
585 emit_move_insn (target, tagged_value);
586 }
587
588 /* This should get expanded in the sanopt pass. */
589
590 static void
591 expand_ASAN_CHECK (internal_fn, gcall *)
592 {
593 gcc_unreachable ();
594 }
595
596 /* This should get expanded in the sanopt pass. */
597
598 static void
599 expand_ASAN_MARK (internal_fn, gcall *)
600 {
601 gcc_unreachable ();
602 }
603
604 /* This should get expanded in the sanopt pass. */
605
606 static void
607 expand_ASAN_POISON (internal_fn, gcall *)
608 {
609 gcc_unreachable ();
610 }
611
612 /* This should get expanded in the sanopt pass. */
613
614 static void
615 expand_ASAN_POISON_USE (internal_fn, gcall *)
616 {
617 gcc_unreachable ();
618 }
619
620 /* This should get expanded in the tsan pass. */
621
622 static void
623 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
624 {
625 gcc_unreachable ();
626 }
627
628 /* This should get expanded in the lower pass. */
629
630 static void
631 expand_FALLTHROUGH (internal_fn, gcall *call)
632 {
633 error_at (gimple_location (call),
634 "invalid use of attribute %<fallthrough%>");
635 }
636
637 /* Return minimum precision needed to represent all values
638 of ARG in SIGNed integral type. */
639
640 static int
641 get_min_precision (tree arg, signop sign)
642 {
643 int prec = TYPE_PRECISION (TREE_TYPE (arg));
644 int cnt = 0;
645 signop orig_sign = sign;
646 if (TREE_CODE (arg) == INTEGER_CST)
647 {
648 int p;
649 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
650 {
651 widest_int w = wi::to_widest (arg);
652 w = wi::ext (w, prec, sign);
653 p = wi::min_precision (w, sign);
654 }
655 else
656 p = wi::min_precision (wi::to_wide (arg), sign);
657 return MIN (p, prec);
658 }
659 while (CONVERT_EXPR_P (arg)
660 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
661 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
662 {
663 arg = TREE_OPERAND (arg, 0);
664 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
665 {
666 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
667 sign = UNSIGNED;
668 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
669 return prec + (orig_sign != sign);
670 prec = TYPE_PRECISION (TREE_TYPE (arg));
671 }
672 if (++cnt > 30)
673 return prec + (orig_sign != sign);
674 }
675 if (CONVERT_EXPR_P (arg)
676 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
677 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
678 {
679 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
680 If y_2's min precision is smaller than prec, return that. */
681 int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
682 if (oprec < prec)
683 return oprec + (orig_sign != sign);
684 }
685 if (TREE_CODE (arg) != SSA_NAME)
686 return prec + (orig_sign != sign);
687 value_range r;
688 while (!get_global_range_query ()->range_of_expr (r, arg)
689 || r.kind () != VR_RANGE)
690 {
691 gimple *g = SSA_NAME_DEF_STMT (arg);
692 if (is_gimple_assign (g)
693 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
694 {
695 tree t = gimple_assign_rhs1 (g);
696 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
697 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
698 {
699 arg = t;
700 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
701 {
702 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
703 sign = UNSIGNED;
704 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
705 return prec + (orig_sign != sign);
706 prec = TYPE_PRECISION (TREE_TYPE (arg));
707 }
708 if (++cnt > 30)
709 return prec + (orig_sign != sign);
710 continue;
711 }
712 }
713 return prec + (orig_sign != sign);
714 }
715 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
716 {
717 int p1 = wi::min_precision (r.lower_bound (), sign);
718 int p2 = wi::min_precision (r.upper_bound (), sign);
719 p1 = MAX (p1, p2);
720 prec = MIN (prec, p1);
721 }
722 else if (sign == UNSIGNED && !wi::neg_p (r.lower_bound (), SIGNED))
723 {
724 int p = wi::min_precision (r.upper_bound (), UNSIGNED);
725 prec = MIN (prec, p);
726 }
727 return prec + (orig_sign != sign);
728 }
729
730 /* Helper for expand_*_overflow. Set the __imag__ part to true
731 (1 except for signed:1 type, in which case store -1). */
732
733 static void
734 expand_arith_set_overflow (tree lhs, rtx target)
735 {
736 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
737 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
738 write_complex_part (target, constm1_rtx, true);
739 else
740 write_complex_part (target, const1_rtx, true);
741 }
742
743 /* Helper for expand_*_overflow. Store RES into the __real__ part
744 of TARGET. If RES has larger MODE than __real__ part of TARGET,
745 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
746 if LHS has smaller precision than its mode. */
747
748 static void
749 expand_arith_overflow_result_store (tree lhs, rtx target,
750 scalar_int_mode mode, rtx res)
751 {
752 scalar_int_mode tgtmode
753 = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
754 rtx lres = res;
755 if (tgtmode != mode)
756 {
757 rtx_code_label *done_label = gen_label_rtx ();
758 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
759 lres = convert_modes (tgtmode, mode, res, uns);
760 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
761 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
762 EQ, true, mode, NULL_RTX, NULL, done_label,
763 profile_probability::very_likely ());
764 expand_arith_set_overflow (lhs, target);
765 emit_label (done_label);
766 }
767 int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
768 int tgtprec = GET_MODE_PRECISION (tgtmode);
769 if (prec < tgtprec)
770 {
771 rtx_code_label *done_label = gen_label_rtx ();
772 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
773 res = lres;
774 if (uns)
775 {
776 rtx mask
777 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
778 tgtmode);
779 lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
780 true, OPTAB_LIB_WIDEN);
781 }
782 else
783 {
784 lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
785 NULL_RTX, 1);
786 lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
787 NULL_RTX, 0);
788 }
789 do_compare_rtx_and_jump (res, lres,
790 EQ, true, tgtmode, NULL_RTX, NULL, done_label,
791 profile_probability::very_likely ());
792 expand_arith_set_overflow (lhs, target);
793 emit_label (done_label);
794 }
795 write_complex_part (target, lres, false);
796 }
797
798 /* Helper for expand_*_overflow. Store RES into TARGET. */
799
800 static void
801 expand_ubsan_result_store (rtx target, rtx res)
802 {
803 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
804 /* If this is a scalar in a register that is stored in a wider mode
805 than the declared mode, compute the result into its declared mode
806 and then convert to the wider mode. Our value is the computed
807 expression. */
808 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
809 else
810 emit_move_insn (target, res);
811 }
812
813 /* Add sub/add overflow checking to the statement STMT.
814 CODE says whether the operation is +, or -. */
815
816 void
817 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
818 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
819 bool uns1_p, bool is_ubsan, tree *datap)
820 {
821 rtx res, target = NULL_RTX;
822 tree fn;
823 rtx_code_label *done_label = gen_label_rtx ();
824 rtx_code_label *do_error = gen_label_rtx ();
825 do_pending_stack_adjust ();
826 rtx op0 = expand_normal (arg0);
827 rtx op1 = expand_normal (arg1);
828 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
829 int prec = GET_MODE_PRECISION (mode);
830 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
831 bool do_xor = false;
832
833 if (is_ubsan)
834 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
835
836 if (lhs)
837 {
838 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
839 if (!is_ubsan)
840 write_complex_part (target, const0_rtx, true);
841 }
842
843 /* We assume both operands and result have the same precision
844 here (GET_MODE_BITSIZE (mode)), S stands for signed type
845 with that precision, U for unsigned type with that precision,
846 sgn for unsigned most significant bit in that precision.
847 s1 is signed first operand, u1 is unsigned first operand,
848 s2 is signed second operand, u2 is unsigned second operand,
849 sr is signed result, ur is unsigned result and the following
850 rules say how to compute result (which is always result of
851 the operands as if both were unsigned, cast to the right
852 signedness) and how to compute whether operation overflowed.
853
854 s1 + s2 -> sr
855 res = (S) ((U) s1 + (U) s2)
856 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
857 s1 - s2 -> sr
858 res = (S) ((U) s1 - (U) s2)
859 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
860 u1 + u2 -> ur
861 res = u1 + u2
862 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
863 u1 - u2 -> ur
864 res = u1 - u2
865 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
866 s1 + u2 -> sr
867 res = (S) ((U) s1 + u2)
868 ovf = ((U) res ^ sgn) < u2
869 s1 + u2 -> ur
870 t1 = (S) (u2 ^ sgn)
871 t2 = s1 + t1
872 res = (U) t2 ^ sgn
873 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
874 s1 - u2 -> sr
875 res = (S) ((U) s1 - u2)
876 ovf = u2 > ((U) s1 ^ sgn)
877 s1 - u2 -> ur
878 res = (U) s1 - u2
879 ovf = s1 < 0 || u2 > (U) s1
880 u1 - s2 -> sr
881 res = u1 - (U) s2
882 ovf = u1 >= ((U) s2 ^ sgn)
883 u1 - s2 -> ur
884 t1 = u1 ^ sgn
885 t2 = t1 - (U) s2
886 res = t2 ^ sgn
887 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
888 s1 + s2 -> ur
889 res = (U) s1 + (U) s2
890 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
891 u1 + u2 -> sr
892 res = (S) (u1 + u2)
893 ovf = (U) res < u2 || res < 0
894 u1 - u2 -> sr
895 res = (S) (u1 - u2)
896 ovf = u1 >= u2 ? res < 0 : res >= 0
897 s1 - s2 -> ur
898 res = (U) s1 - (U) s2
899 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
900
901 if (code == PLUS_EXPR && uns0_p && !uns1_p)
902 {
903 /* PLUS_EXPR is commutative, if operand signedness differs,
904 canonicalize to the first operand being signed and second
905 unsigned to simplify following code. */
906 std::swap (op0, op1);
907 std::swap (arg0, arg1);
908 uns0_p = false;
909 uns1_p = true;
910 }
911
912 /* u1 +- u2 -> ur */
913 if (uns0_p && uns1_p && unsr_p)
914 {
915 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
916 : usubv4_optab, mode);
917 if (icode != CODE_FOR_nothing)
918 {
919 class expand_operand ops[4];
920 rtx_insn *last = get_last_insn ();
921
922 res = gen_reg_rtx (mode);
923 create_output_operand (&ops[0], res, mode);
924 create_input_operand (&ops[1], op0, mode);
925 create_input_operand (&ops[2], op1, mode);
926 create_fixed_operand (&ops[3], do_error);
927 if (maybe_expand_insn (icode, 4, ops))
928 {
929 last = get_last_insn ();
930 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
931 && JUMP_P (last)
932 && any_condjump_p (last)
933 && !find_reg_note (last, REG_BR_PROB, 0))
934 add_reg_br_prob_note (last,
935 profile_probability::very_unlikely ());
936 emit_jump (done_label);
937 goto do_error_label;
938 }
939
940 delete_insns_since (last);
941 }
942
943 /* Compute the operation. On RTL level, the addition is always
944 unsigned. */
945 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
946 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
947 rtx tem = op0;
948 /* For PLUS_EXPR, the operation is commutative, so we can pick
949 operand to compare against. For prec <= BITS_PER_WORD, I think
950 preferring REG operand is better over CONST_INT, because
951 the CONST_INT might enlarge the instruction or CSE would need
952 to figure out we'd already loaded it into a register before.
953 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
954 as then the multi-word comparison can be perhaps simplified. */
955 if (code == PLUS_EXPR
956 && (prec <= BITS_PER_WORD
957 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
958 : CONST_SCALAR_INT_P (op1)))
959 tem = op1;
960 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
961 true, mode, NULL_RTX, NULL, done_label,
962 profile_probability::very_likely ());
963 goto do_error_label;
964 }
965
966 /* s1 +- u2 -> sr */
967 if (!uns0_p && uns1_p && !unsr_p)
968 {
969 /* Compute the operation. On RTL level, the addition is always
970 unsigned. */
971 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
972 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
973 rtx tem = expand_binop (mode, add_optab,
974 code == PLUS_EXPR ? res : op0, sgn,
975 NULL_RTX, false, OPTAB_LIB_WIDEN);
976 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
977 done_label, profile_probability::very_likely ());
978 goto do_error_label;
979 }
980
981 /* s1 + u2 -> ur */
982 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
983 {
984 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
985 OPTAB_LIB_WIDEN);
986 /* As we've changed op1, we have to avoid using the value range
987 for the original argument. */
988 arg1 = error_mark_node;
989 do_xor = true;
990 goto do_signed;
991 }
992
993 /* u1 - s2 -> ur */
994 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
995 {
996 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
997 OPTAB_LIB_WIDEN);
998 /* As we've changed op0, we have to avoid using the value range
999 for the original argument. */
1000 arg0 = error_mark_node;
1001 do_xor = true;
1002 goto do_signed;
1003 }
1004
1005 /* s1 - u2 -> ur */
1006 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
1007 {
1008 /* Compute the operation. On RTL level, the addition is always
1009 unsigned. */
1010 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1011 OPTAB_LIB_WIDEN);
1012 int pos_neg = get_range_pos_neg (arg0);
1013 if (pos_neg == 2)
1014 /* If ARG0 is known to be always negative, this is always overflow. */
1015 emit_jump (do_error);
1016 else if (pos_neg == 3)
1017 /* If ARG0 is not known to be always positive, check at runtime. */
1018 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
1019 NULL, do_error, profile_probability::very_unlikely ());
1020 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
1021 done_label, profile_probability::very_likely ());
1022 goto do_error_label;
1023 }
1024
1025 /* u1 - s2 -> sr */
1026 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
1027 {
1028 /* Compute the operation. On RTL level, the addition is always
1029 unsigned. */
1030 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1031 OPTAB_LIB_WIDEN);
1032 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1033 OPTAB_LIB_WIDEN);
1034 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
1035 done_label, profile_probability::very_likely ());
1036 goto do_error_label;
1037 }
1038
1039 /* u1 + u2 -> sr */
1040 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
1041 {
1042 /* Compute the operation. On RTL level, the addition is always
1043 unsigned. */
1044 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
1045 OPTAB_LIB_WIDEN);
1046 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1047 NULL, do_error, profile_probability::very_unlikely ());
1048 rtx tem = op1;
1049 /* The operation is commutative, so we can pick operand to compare
1050 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1051 is better over CONST_INT, because the CONST_INT might enlarge the
1052 instruction or CSE would need to figure out we'd already loaded it
1053 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1054 might be more beneficial, as then the multi-word comparison can be
1055 perhaps simplified. */
1056 if (prec <= BITS_PER_WORD
1057 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
1058 : CONST_SCALAR_INT_P (op0))
1059 tem = op0;
1060 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
1061 done_label, profile_probability::very_likely ());
1062 goto do_error_label;
1063 }
1064
1065 /* s1 +- s2 -> ur */
1066 if (!uns0_p && !uns1_p && unsr_p)
1067 {
1068 /* Compute the operation. On RTL level, the addition is always
1069 unsigned. */
1070 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1071 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1072 int pos_neg = get_range_pos_neg (arg1);
1073 if (code == PLUS_EXPR)
1074 {
1075 int pos_neg0 = get_range_pos_neg (arg0);
1076 if (pos_neg0 != 3 && pos_neg == 3)
1077 {
1078 std::swap (op0, op1);
1079 pos_neg = pos_neg0;
1080 }
1081 }
1082 rtx tem;
1083 if (pos_neg != 3)
1084 {
1085 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
1086 ? and_optab : ior_optab,
1087 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
1088 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
1089 NULL, done_label, profile_probability::very_likely ());
1090 }
1091 else
1092 {
1093 rtx_code_label *do_ior_label = gen_label_rtx ();
1094 do_compare_rtx_and_jump (op1, const0_rtx,
1095 code == MINUS_EXPR ? GE : LT, false, mode,
1096 NULL_RTX, NULL, do_ior_label,
1097 profile_probability::even ());
1098 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
1099 OPTAB_LIB_WIDEN);
1100 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1101 NULL, done_label, profile_probability::very_likely ());
1102 emit_jump (do_error);
1103 emit_label (do_ior_label);
1104 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
1105 OPTAB_LIB_WIDEN);
1106 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1107 NULL, done_label, profile_probability::very_likely ());
1108 }
1109 goto do_error_label;
1110 }
1111
1112 /* u1 - u2 -> sr */
1113 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
1114 {
1115 /* Compute the operation. On RTL level, the addition is always
1116 unsigned. */
1117 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1118 OPTAB_LIB_WIDEN);
1119 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
1120 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
1121 op0_geu_op1, profile_probability::even ());
1122 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1123 NULL, done_label, profile_probability::very_likely ());
1124 emit_jump (do_error);
1125 emit_label (op0_geu_op1);
1126 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1127 NULL, done_label, profile_probability::very_likely ());
1128 goto do_error_label;
1129 }
1130
1131 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
1132
1133 /* s1 +- s2 -> sr */
1134 do_signed:
1135 {
1136 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
1137 : subv4_optab, mode);
1138 if (icode != CODE_FOR_nothing)
1139 {
1140 class expand_operand ops[4];
1141 rtx_insn *last = get_last_insn ();
1142
1143 res = gen_reg_rtx (mode);
1144 create_output_operand (&ops[0], res, mode);
1145 create_input_operand (&ops[1], op0, mode);
1146 create_input_operand (&ops[2], op1, mode);
1147 create_fixed_operand (&ops[3], do_error);
1148 if (maybe_expand_insn (icode, 4, ops))
1149 {
1150 last = get_last_insn ();
1151 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1152 && JUMP_P (last)
1153 && any_condjump_p (last)
1154 && !find_reg_note (last, REG_BR_PROB, 0))
1155 add_reg_br_prob_note (last,
1156 profile_probability::very_unlikely ());
1157 emit_jump (done_label);
1158 goto do_error_label;
1159 }
1160
1161 delete_insns_since (last);
1162 }
1163
1164 /* Compute the operation. On RTL level, the addition is always
1165 unsigned. */
1166 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1167 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1168
1169 /* If we can prove that one of the arguments (for MINUS_EXPR only
1170 the second operand, as subtraction is not commutative) is always
1171 non-negative or always negative, we can do just one comparison
1172 and conditional jump. */
1173 int pos_neg = get_range_pos_neg (arg1);
1174 if (code == PLUS_EXPR)
1175 {
1176 int pos_neg0 = get_range_pos_neg (arg0);
1177 if (pos_neg0 != 3 && pos_neg == 3)
1178 {
1179 std::swap (op0, op1);
1180 pos_neg = pos_neg0;
1181 }
1182 }
1183
1184 /* Addition overflows if and only if the two operands have the same sign,
1185 and the result has the opposite sign. Subtraction overflows if and
1186 only if the two operands have opposite sign, and the subtrahend has
1187 the same sign as the result. Here 0 is counted as positive. */
1188 if (pos_neg == 3)
1189 {
1190 /* Compute op0 ^ op1 (operands have opposite sign). */
1191 rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1192 OPTAB_LIB_WIDEN);
1193
1194 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1195 rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1196 OPTAB_LIB_WIDEN);
1197
1198 rtx tem;
1199 if (code == PLUS_EXPR)
1200 {
1201 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1202 tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1203 tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1204 OPTAB_LIB_WIDEN);
1205 }
1206 else
1207 {
1208 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1209 tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1210 tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1211 OPTAB_LIB_WIDEN);
1212 }
1213
1214 /* No overflow if the result has bit sign cleared. */
1215 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1216 NULL, done_label, profile_probability::very_likely ());
1217 }
1218
1219 /* Compare the result of the operation with the first operand.
1220 No overflow for addition if second operand is positive and result
1221 is larger or second operand is negative and result is smaller.
1222 Likewise for subtraction with sign of second operand flipped. */
1223 else
1224 do_compare_rtx_and_jump (res, op0,
1225 (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1226 false, mode, NULL_RTX, NULL, done_label,
1227 profile_probability::very_likely ());
1228 }
1229
1230 do_error_label:
1231 emit_label (do_error);
1232 if (is_ubsan)
1233 {
1234 /* Expand the ubsan builtin call. */
1235 push_temp_slots ();
1236 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1237 arg0, arg1, datap);
1238 expand_normal (fn);
1239 pop_temp_slots ();
1240 do_pending_stack_adjust ();
1241 }
1242 else if (lhs)
1243 expand_arith_set_overflow (lhs, target);
1244
1245 /* We're done. */
1246 emit_label (done_label);
1247
1248 if (lhs)
1249 {
1250 if (is_ubsan)
1251 expand_ubsan_result_store (target, res);
1252 else
1253 {
1254 if (do_xor)
1255 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1256 OPTAB_LIB_WIDEN);
1257
1258 expand_arith_overflow_result_store (lhs, target, mode, res);
1259 }
1260 }
1261 }
1262
1263 /* Add negate overflow checking to the statement STMT. */
1264
1265 static void
1266 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1267 tree *datap)
1268 {
1269 rtx res, op1;
1270 tree fn;
1271 rtx_code_label *done_label, *do_error;
1272 rtx target = NULL_RTX;
1273
1274 done_label = gen_label_rtx ();
1275 do_error = gen_label_rtx ();
1276
1277 do_pending_stack_adjust ();
1278 op1 = expand_normal (arg1);
1279
1280 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1281 if (lhs)
1282 {
1283 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1284 if (!is_ubsan)
1285 write_complex_part (target, const0_rtx, true);
1286 }
1287
1288 enum insn_code icode = optab_handler (negv3_optab, mode);
1289 if (icode != CODE_FOR_nothing)
1290 {
1291 class expand_operand ops[3];
1292 rtx_insn *last = get_last_insn ();
1293
1294 res = gen_reg_rtx (mode);
1295 create_output_operand (&ops[0], res, mode);
1296 create_input_operand (&ops[1], op1, mode);
1297 create_fixed_operand (&ops[2], do_error);
1298 if (maybe_expand_insn (icode, 3, ops))
1299 {
1300 last = get_last_insn ();
1301 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1302 && JUMP_P (last)
1303 && any_condjump_p (last)
1304 && !find_reg_note (last, REG_BR_PROB, 0))
1305 add_reg_br_prob_note (last,
1306 profile_probability::very_unlikely ());
1307 emit_jump (done_label);
1308 }
1309 else
1310 {
1311 delete_insns_since (last);
1312 icode = CODE_FOR_nothing;
1313 }
1314 }
1315
1316 if (icode == CODE_FOR_nothing)
1317 {
1318 /* Compute the operation. On RTL level, the addition is always
1319 unsigned. */
1320 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1321
1322 /* Compare the operand with the most negative value. */
1323 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1324 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1325 done_label, profile_probability::very_likely ());
1326 }
1327
1328 emit_label (do_error);
1329 if (is_ubsan)
1330 {
1331 /* Expand the ubsan builtin call. */
1332 push_temp_slots ();
1333 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1334 arg1, NULL_TREE, datap);
1335 expand_normal (fn);
1336 pop_temp_slots ();
1337 do_pending_stack_adjust ();
1338 }
1339 else if (lhs)
1340 expand_arith_set_overflow (lhs, target);
1341
1342 /* We're done. */
1343 emit_label (done_label);
1344
1345 if (lhs)
1346 {
1347 if (is_ubsan)
1348 expand_ubsan_result_store (target, res);
1349 else
1350 expand_arith_overflow_result_store (lhs, target, mode, res);
1351 }
1352 }
1353
1354 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1355 mode MODE can be expanded without using a libcall. */
1356
1357 static bool
1358 can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1359 rtx op0, rtx op1, bool uns)
1360 {
1361 if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1362 != CODE_FOR_nothing)
1363 return true;
1364
1365 if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1366 != CODE_FOR_nothing)
1367 return true;
1368
1369 rtx_insn *last = get_last_insn ();
1370 if (CONSTANT_P (op0))
1371 op0 = convert_modes (wmode, mode, op0, uns);
1372 else
1373 op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1374 if (CONSTANT_P (op1))
1375 op1 = convert_modes (wmode, mode, op1, uns);
1376 else
1377 op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1378 rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1379 delete_insns_since (last);
1380 return ret != NULL_RTX;
1381 }
1382
1383 /* Add mul overflow checking to the statement STMT. */
1384
1385 static void
1386 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1387 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1388 tree *datap)
1389 {
1390 rtx res, op0, op1;
1391 tree fn, type;
1392 rtx_code_label *done_label, *do_error;
1393 rtx target = NULL_RTX;
1394 signop sign;
1395 enum insn_code icode;
1396
1397 done_label = gen_label_rtx ();
1398 do_error = gen_label_rtx ();
1399
1400 do_pending_stack_adjust ();
1401 op0 = expand_normal (arg0);
1402 op1 = expand_normal (arg1);
1403
1404 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1405 bool uns = unsr_p;
1406 if (lhs)
1407 {
1408 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1409 if (!is_ubsan)
1410 write_complex_part (target, const0_rtx, true);
1411 }
1412
1413 if (is_ubsan)
1414 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1415
1416 /* We assume both operands and result have the same precision
1417 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1418 with that precision, U for unsigned type with that precision,
1419 sgn for unsigned most significant bit in that precision.
1420 s1 is signed first operand, u1 is unsigned first operand,
1421 s2 is signed second operand, u2 is unsigned second operand,
1422 sr is signed result, ur is unsigned result and the following
1423 rules say how to compute result (which is always result of
1424 the operands as if both were unsigned, cast to the right
1425 signedness) and how to compute whether operation overflowed.
1426 main_ovf (false) stands for jump on signed multiplication
1427 overflow or the main algorithm with uns == false.
1428 main_ovf (true) stands for jump on unsigned multiplication
1429 overflow or the main algorithm with uns == true.
1430
1431 s1 * s2 -> sr
1432 res = (S) ((U) s1 * (U) s2)
1433 ovf = main_ovf (false)
1434 u1 * u2 -> ur
1435 res = u1 * u2
1436 ovf = main_ovf (true)
1437 s1 * u2 -> ur
1438 res = (U) s1 * u2
1439 ovf = (s1 < 0 && u2) || main_ovf (true)
1440 u1 * u2 -> sr
1441 res = (S) (u1 * u2)
1442 ovf = res < 0 || main_ovf (true)
1443 s1 * u2 -> sr
1444 res = (S) ((U) s1 * u2)
1445 ovf = (S) u2 >= 0 ? main_ovf (false)
1446 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1447 s1 * s2 -> ur
1448 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1449 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1450 res = t1 * t2
1451 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1452
1453 if (uns0_p && !uns1_p)
1454 {
1455 /* Multiplication is commutative, if operand signedness differs,
1456 canonicalize to the first operand being signed and second
1457 unsigned to simplify following code. */
1458 std::swap (op0, op1);
1459 std::swap (arg0, arg1);
1460 uns0_p = false;
1461 uns1_p = true;
1462 }
1463
1464 int pos_neg0 = get_range_pos_neg (arg0);
1465 int pos_neg1 = get_range_pos_neg (arg1);
1466
1467 /* s1 * u2 -> ur */
1468 if (!uns0_p && uns1_p && unsr_p)
1469 {
1470 switch (pos_neg0)
1471 {
1472 case 1:
1473 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1474 goto do_main;
1475 case 2:
1476 /* If s1 is negative, avoid the main code, just multiply and
1477 signal overflow if op1 is not 0. */
1478 struct separate_ops ops;
1479 ops.code = MULT_EXPR;
1480 ops.type = TREE_TYPE (arg1);
1481 ops.op0 = make_tree (ops.type, op0);
1482 ops.op1 = make_tree (ops.type, op1);
1483 ops.op2 = NULL_TREE;
1484 ops.location = loc;
1485 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1486 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1487 NULL, done_label, profile_probability::very_likely ());
1488 goto do_error_label;
1489 case 3:
1490 if (get_min_precision (arg1, UNSIGNED)
1491 + get_min_precision (arg0, SIGNED) <= GET_MODE_PRECISION (mode))
1492 {
1493 /* If the first operand is sign extended from narrower type, the
1494 second operand is zero extended from narrower type and
1495 the sum of the two precisions is smaller or equal to the
1496 result precision: if the first argument is at runtime
1497 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1498 and there will be no overflow, if the first argument is
1499 negative and the second argument zero, the result will be
1500 0 and there will be no overflow, if the first argument is
1501 negative and the second argument positive, the result when
1502 treated as signed will be negative (minimum -0x7f80 or
1503 -0x7f..f80..0) there will be always overflow. So, do
1504 res = (U) (s1 * u2)
1505 ovf = (S) res < 0 */
1506 struct separate_ops ops;
1507 ops.code = MULT_EXPR;
1508 ops.type
1509 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1510 1);
1511 ops.op0 = make_tree (ops.type, op0);
1512 ops.op1 = make_tree (ops.type, op1);
1513 ops.op2 = NULL_TREE;
1514 ops.location = loc;
1515 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1516 do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1517 mode, NULL_RTX, NULL, done_label,
1518 profile_probability::very_likely ());
1519 goto do_error_label;
1520 }
1521 rtx_code_label *do_main_label;
1522 do_main_label = gen_label_rtx ();
1523 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1524 NULL, do_main_label, profile_probability::very_likely ());
1525 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1526 NULL, do_main_label, profile_probability::very_likely ());
1527 expand_arith_set_overflow (lhs, target);
1528 emit_label (do_main_label);
1529 goto do_main;
1530 default:
1531 gcc_unreachable ();
1532 }
1533 }
1534
1535 /* u1 * u2 -> sr */
1536 if (uns0_p && uns1_p && !unsr_p)
1537 {
1538 if ((pos_neg0 | pos_neg1) == 1)
1539 {
1540 /* If both arguments are zero extended from narrower types,
1541 the MSB will be clear on both and so we can pretend it is
1542 a normal s1 * s2 -> sr multiplication. */
1543 uns0_p = false;
1544 uns1_p = false;
1545 }
1546 else
1547 uns = true;
1548 /* Rest of handling of this case after res is computed. */
1549 goto do_main;
1550 }
1551
1552 /* s1 * u2 -> sr */
1553 if (!uns0_p && uns1_p && !unsr_p)
1554 {
1555 switch (pos_neg1)
1556 {
1557 case 1:
1558 goto do_main;
1559 case 2:
1560 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1561 avoid the main code, just multiply and signal overflow
1562 unless 0 * u2 or -1 * ((U) Smin). */
1563 struct separate_ops ops;
1564 ops.code = MULT_EXPR;
1565 ops.type = TREE_TYPE (arg1);
1566 ops.op0 = make_tree (ops.type, op0);
1567 ops.op1 = make_tree (ops.type, op1);
1568 ops.op2 = NULL_TREE;
1569 ops.location = loc;
1570 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1571 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1572 NULL, done_label, profile_probability::very_likely ());
1573 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1574 NULL, do_error, profile_probability::very_unlikely ());
1575 int prec;
1576 prec = GET_MODE_PRECISION (mode);
1577 rtx sgn;
1578 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1579 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1580 NULL, done_label, profile_probability::very_likely ());
1581 goto do_error_label;
1582 case 3:
1583 /* Rest of handling of this case after res is computed. */
1584 goto do_main;
1585 default:
1586 gcc_unreachable ();
1587 }
1588 }
1589
1590 /* s1 * s2 -> ur */
1591 if (!uns0_p && !uns1_p && unsr_p)
1592 {
1593 rtx tem;
1594 switch (pos_neg0 | pos_neg1)
1595 {
1596 case 1: /* Both operands known to be non-negative. */
1597 goto do_main;
1598 case 2: /* Both operands known to be negative. */
1599 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1600 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1601 /* Avoid looking at arg0/arg1 ranges, as we've changed
1602 the arguments. */
1603 arg0 = error_mark_node;
1604 arg1 = error_mark_node;
1605 goto do_main;
1606 case 3:
1607 if ((pos_neg0 ^ pos_neg1) == 3)
1608 {
1609 /* If one operand is known to be negative and the other
1610 non-negative, this overflows always, unless the non-negative
1611 one is 0. Just do normal multiply and set overflow
1612 unless one of the operands is 0. */
1613 struct separate_ops ops;
1614 ops.code = MULT_EXPR;
1615 ops.type
1616 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1617 1);
1618 ops.op0 = make_tree (ops.type, op0);
1619 ops.op1 = make_tree (ops.type, op1);
1620 ops.op2 = NULL_TREE;
1621 ops.location = loc;
1622 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1623 do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1624 true, mode, NULL_RTX, NULL, done_label,
1625 profile_probability::very_likely ());
1626 goto do_error_label;
1627 }
1628 if (get_min_precision (arg0, SIGNED)
1629 + get_min_precision (arg1, SIGNED) <= GET_MODE_PRECISION (mode))
1630 {
1631 /* If both operands are sign extended from narrower types and
1632 the sum of the two precisions is smaller or equal to the
1633 result precision: if both arguments are at runtime
1634 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1635 and there will be no overflow, if both arguments are negative,
1636 maximum result will be 0x40..00 and there will be no overflow
1637 either, if one argument is positive and the other argument
1638 negative, the result when treated as signed will be negative
1639 and there will be always overflow, and if one argument is
1640 zero and the other negative the result will be zero and no
1641 overflow. So, do
1642 res = (U) (s1 * s2)
1643 ovf = (S) res < 0 */
1644 struct separate_ops ops;
1645 ops.code = MULT_EXPR;
1646 ops.type
1647 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1648 1);
1649 ops.op0 = make_tree (ops.type, op0);
1650 ops.op1 = make_tree (ops.type, op1);
1651 ops.op2 = NULL_TREE;
1652 ops.location = loc;
1653 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1654 do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1655 mode, NULL_RTX, NULL, done_label,
1656 profile_probability::very_likely ());
1657 goto do_error_label;
1658 }
1659 /* The general case, do all the needed comparisons at runtime. */
1660 rtx_code_label *do_main_label, *after_negate_label;
1661 rtx rop0, rop1;
1662 rop0 = gen_reg_rtx (mode);
1663 rop1 = gen_reg_rtx (mode);
1664 emit_move_insn (rop0, op0);
1665 emit_move_insn (rop1, op1);
1666 op0 = rop0;
1667 op1 = rop1;
1668 do_main_label = gen_label_rtx ();
1669 after_negate_label = gen_label_rtx ();
1670 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1671 OPTAB_LIB_WIDEN);
1672 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1673 NULL, after_negate_label, profile_probability::very_likely ());
1674 /* Both arguments negative here, negate them and continue with
1675 normal unsigned overflow checking multiplication. */
1676 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1677 NULL_RTX, false));
1678 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1679 NULL_RTX, false));
1680 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1681 the arguments. */
1682 arg0 = error_mark_node;
1683 arg1 = error_mark_node;
1684 emit_jump (do_main_label);
1685 emit_label (after_negate_label);
1686 tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1687 OPTAB_LIB_WIDEN);
1688 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1689 NULL, do_main_label,
1690 profile_probability::very_likely ());
1691 /* One argument is negative here, the other positive. This
1692 overflows always, unless one of the arguments is 0. But
1693 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1694 is, thus we can keep do_main code oring in overflow as is. */
1695 if (pos_neg0 != 2)
1696 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1697 NULL, do_main_label,
1698 profile_probability::very_unlikely ());
1699 if (pos_neg1 != 2)
1700 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1701 NULL, do_main_label,
1702 profile_probability::very_unlikely ());
1703 expand_arith_set_overflow (lhs, target);
1704 emit_label (do_main_label);
1705 goto do_main;
1706 default:
1707 gcc_unreachable ();
1708 }
1709 }
1710
1711 do_main:
1712 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1713 sign = uns ? UNSIGNED : SIGNED;
1714 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
1715 if (uns
1716 && (integer_pow2p (arg0) || integer_pow2p (arg1))
1717 && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
1718 {
1719 /* Optimize unsigned multiplication by power of 2 constant
1720 using 2 shifts, one for result, one to extract the shifted
1721 out bits to see if they are all zero.
1722 Don't do this if optimizing for size and we have umulv4_optab,
1723 in that case assume multiplication will be shorter.
1724 This is heuristics based on the single target that provides
1725 umulv4 right now (i?86/x86_64), if further targets add it, this
1726 might need to be revisited.
1727 Cases where both operands are constant should be folded already
1728 during GIMPLE, and cases where one operand is constant but not
1729 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1730 below can be done without multiplication, just by shifts and adds,
1731 or we'd need to divide the result (and hope it actually doesn't
1732 really divide nor multiply) and compare the result of the division
1733 with the original operand. */
1734 rtx opn0 = op0;
1735 rtx opn1 = op1;
1736 tree argn0 = arg0;
1737 tree argn1 = arg1;
1738 if (integer_pow2p (arg0))
1739 {
1740 std::swap (opn0, opn1);
1741 std::swap (argn0, argn1);
1742 }
1743 int cnt = tree_log2 (argn1);
1744 if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
1745 {
1746 rtx upper = const0_rtx;
1747 res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
1748 if (cnt != 0)
1749 upper = expand_shift (RSHIFT_EXPR, mode, opn0,
1750 GET_MODE_PRECISION (mode) - cnt,
1751 NULL_RTX, uns);
1752 do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
1753 NULL_RTX, NULL, done_label,
1754 profile_probability::very_likely ());
1755 goto do_error_label;
1756 }
1757 }
1758 if (icode != CODE_FOR_nothing)
1759 {
1760 class expand_operand ops[4];
1761 rtx_insn *last = get_last_insn ();
1762
1763 res = gen_reg_rtx (mode);
1764 create_output_operand (&ops[0], res, mode);
1765 create_input_operand (&ops[1], op0, mode);
1766 create_input_operand (&ops[2], op1, mode);
1767 create_fixed_operand (&ops[3], do_error);
1768 if (maybe_expand_insn (icode, 4, ops))
1769 {
1770 last = get_last_insn ();
1771 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1772 && JUMP_P (last)
1773 && any_condjump_p (last)
1774 && !find_reg_note (last, REG_BR_PROB, 0))
1775 add_reg_br_prob_note (last,
1776 profile_probability::very_unlikely ());
1777 emit_jump (done_label);
1778 }
1779 else
1780 {
1781 delete_insns_since (last);
1782 icode = CODE_FOR_nothing;
1783 }
1784 }
1785
1786 if (icode == CODE_FOR_nothing)
1787 {
1788 struct separate_ops ops;
1789 int prec = GET_MODE_PRECISION (mode);
1790 scalar_int_mode hmode, wmode;
1791 ops.op0 = make_tree (type, op0);
1792 ops.op1 = make_tree (type, op1);
1793 ops.op2 = NULL_TREE;
1794 ops.location = loc;
1795
1796 /* Optimize unsigned overflow check where we don't use the
1797 multiplication result, just whether overflow happened.
1798 If we can do MULT_HIGHPART_EXPR, that followed by
1799 comparison of the result against zero is cheapest.
1800 We'll still compute res, but it should be DCEd later. */
1801 use_operand_p use;
1802 gimple *use_stmt;
1803 if (!is_ubsan
1804 && lhs
1805 && uns
1806 && !(uns0_p && uns1_p && !unsr_p)
1807 && can_mult_highpart_p (mode, uns) == 1
1808 && single_imm_use (lhs, &use, &use_stmt)
1809 && is_gimple_assign (use_stmt)
1810 && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
1811 goto highpart;
1812
1813 if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
1814 && targetm.scalar_mode_supported_p (wmode)
1815 && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
1816 {
1817 twoxwider:
1818 ops.code = WIDEN_MULT_EXPR;
1819 ops.type
1820 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
1821
1822 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
1823 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
1824 NULL_RTX, uns);
1825 hipart = convert_modes (mode, wmode, hipart, uns);
1826 res = convert_modes (mode, wmode, res, uns);
1827 if (uns)
1828 /* For the unsigned multiplication, there was overflow if
1829 HIPART is non-zero. */
1830 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1831 NULL_RTX, NULL, done_label,
1832 profile_probability::very_likely ());
1833 else
1834 {
1835 /* RES is used more than once, place it in a pseudo. */
1836 res = force_reg (mode, res);
1837
1838 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1839 NULL_RTX, 0);
1840 /* RES is low half of the double width result, HIPART
1841 the high half. There was overflow if
1842 HIPART is different from RES < 0 ? -1 : 0. */
1843 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1844 NULL_RTX, NULL, done_label,
1845 profile_probability::very_likely ());
1846 }
1847 }
1848 else if (can_mult_highpart_p (mode, uns) == 1)
1849 {
1850 highpart:
1851 ops.code = MULT_HIGHPART_EXPR;
1852 ops.type = type;
1853
1854 rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
1855 EXPAND_NORMAL);
1856 ops.code = MULT_EXPR;
1857 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1858 if (uns)
1859 /* For the unsigned multiplication, there was overflow if
1860 HIPART is non-zero. */
1861 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1862 NULL_RTX, NULL, done_label,
1863 profile_probability::very_likely ());
1864 else
1865 {
1866 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1867 NULL_RTX, 0);
1868 /* RES is low half of the double width result, HIPART
1869 the high half. There was overflow if
1870 HIPART is different from RES < 0 ? -1 : 0. */
1871 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1872 NULL_RTX, NULL, done_label,
1873 profile_probability::very_likely ());
1874 }
1875
1876 }
1877 else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
1878 && 2 * GET_MODE_PRECISION (hmode) == prec)
1879 {
1880 rtx_code_label *large_op0 = gen_label_rtx ();
1881 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
1882 rtx_code_label *one_small_one_large = gen_label_rtx ();
1883 rtx_code_label *both_ops_large = gen_label_rtx ();
1884 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
1885 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
1886 rtx_code_label *do_overflow = gen_label_rtx ();
1887 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
1888
1889 unsigned int hprec = GET_MODE_PRECISION (hmode);
1890 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
1891 NULL_RTX, uns);
1892 hipart0 = convert_modes (hmode, mode, hipart0, uns);
1893 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
1894 rtx signbit0 = const0_rtx;
1895 if (!uns)
1896 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
1897 NULL_RTX, 0);
1898 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
1899 NULL_RTX, uns);
1900 hipart1 = convert_modes (hmode, mode, hipart1, uns);
1901 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
1902 rtx signbit1 = const0_rtx;
1903 if (!uns)
1904 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
1905 NULL_RTX, 0);
1906
1907 res = gen_reg_rtx (mode);
1908
1909 /* True if op0 resp. op1 are known to be in the range of
1910 halfstype. */
1911 bool op0_small_p = false;
1912 bool op1_small_p = false;
1913 /* True if op0 resp. op1 are known to have all zeros or all ones
1914 in the upper half of bits, but are not known to be
1915 op{0,1}_small_p. */
1916 bool op0_medium_p = false;
1917 bool op1_medium_p = false;
1918 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1919 nonnegative, 1 if unknown. */
1920 int op0_sign = 1;
1921 int op1_sign = 1;
1922
1923 if (pos_neg0 == 1)
1924 op0_sign = 0;
1925 else if (pos_neg0 == 2)
1926 op0_sign = -1;
1927 if (pos_neg1 == 1)
1928 op1_sign = 0;
1929 else if (pos_neg1 == 2)
1930 op1_sign = -1;
1931
1932 unsigned int mprec0 = prec;
1933 if (arg0 != error_mark_node)
1934 mprec0 = get_min_precision (arg0, sign);
1935 if (mprec0 <= hprec)
1936 op0_small_p = true;
1937 else if (!uns && mprec0 <= hprec + 1)
1938 op0_medium_p = true;
1939 unsigned int mprec1 = prec;
1940 if (arg1 != error_mark_node)
1941 mprec1 = get_min_precision (arg1, sign);
1942 if (mprec1 <= hprec)
1943 op1_small_p = true;
1944 else if (!uns && mprec1 <= hprec + 1)
1945 op1_medium_p = true;
1946
1947 int smaller_sign = 1;
1948 int larger_sign = 1;
1949 if (op0_small_p)
1950 {
1951 smaller_sign = op0_sign;
1952 larger_sign = op1_sign;
1953 }
1954 else if (op1_small_p)
1955 {
1956 smaller_sign = op1_sign;
1957 larger_sign = op0_sign;
1958 }
1959 else if (op0_sign == op1_sign)
1960 {
1961 smaller_sign = op0_sign;
1962 larger_sign = op0_sign;
1963 }
1964
1965 if (!op0_small_p)
1966 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
1967 NULL_RTX, NULL, large_op0,
1968 profile_probability::unlikely ());
1969
1970 if (!op1_small_p)
1971 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1972 NULL_RTX, NULL, small_op0_large_op1,
1973 profile_probability::unlikely ());
1974
1975 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1976 hmode to mode, the multiplication will never overflow. We can
1977 do just one hmode x hmode => mode widening multiplication. */
1978 tree halfstype = build_nonstandard_integer_type (hprec, uns);
1979 ops.op0 = make_tree (halfstype, lopart0);
1980 ops.op1 = make_tree (halfstype, lopart1);
1981 ops.code = WIDEN_MULT_EXPR;
1982 ops.type = type;
1983 rtx thisres
1984 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1985 emit_move_insn (res, thisres);
1986 emit_jump (done_label);
1987
1988 emit_label (small_op0_large_op1);
1989
1990 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1991 but op1 is not, just swap the arguments and handle it as op1
1992 sign/zero extended, op0 not. */
1993 rtx larger = gen_reg_rtx (mode);
1994 rtx hipart = gen_reg_rtx (hmode);
1995 rtx lopart = gen_reg_rtx (hmode);
1996 emit_move_insn (larger, op1);
1997 emit_move_insn (hipart, hipart1);
1998 emit_move_insn (lopart, lopart0);
1999 emit_jump (one_small_one_large);
2000
2001 emit_label (large_op0);
2002
2003 if (!op1_small_p)
2004 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2005 NULL_RTX, NULL, both_ops_large,
2006 profile_probability::unlikely ());
2007
2008 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2009 but op0 is not, prepare larger, hipart and lopart pseudos and
2010 handle it together with small_op0_large_op1. */
2011 emit_move_insn (larger, op0);
2012 emit_move_insn (hipart, hipart0);
2013 emit_move_insn (lopart, lopart1);
2014
2015 emit_label (one_small_one_large);
2016
2017 /* lopart is the low part of the operand that is sign extended
2018 to mode, larger is the other operand, hipart is the
2019 high part of larger and lopart0 and lopart1 are the low parts
2020 of both operands.
2021 We perform lopart0 * lopart1 and lopart * hipart widening
2022 multiplications. */
2023 tree halfutype = build_nonstandard_integer_type (hprec, 1);
2024 ops.op0 = make_tree (halfutype, lopart0);
2025 ops.op1 = make_tree (halfutype, lopart1);
2026 rtx lo0xlo1
2027 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2028
2029 ops.op0 = make_tree (halfutype, lopart);
2030 ops.op1 = make_tree (halfutype, hipart);
2031 rtx loxhi = gen_reg_rtx (mode);
2032 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2033 emit_move_insn (loxhi, tem);
2034
2035 if (!uns)
2036 {
2037 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2038 if (larger_sign == 0)
2039 emit_jump (after_hipart_neg);
2040 else if (larger_sign != -1)
2041 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
2042 NULL_RTX, NULL, after_hipart_neg,
2043 profile_probability::even ());
2044
2045 tem = convert_modes (mode, hmode, lopart, 1);
2046 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
2047 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
2048 1, OPTAB_WIDEN);
2049 emit_move_insn (loxhi, tem);
2050
2051 emit_label (after_hipart_neg);
2052
2053 /* if (lopart < 0) loxhi -= larger; */
2054 if (smaller_sign == 0)
2055 emit_jump (after_lopart_neg);
2056 else if (smaller_sign != -1)
2057 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
2058 NULL_RTX, NULL, after_lopart_neg,
2059 profile_probability::even ());
2060
2061 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
2062 1, OPTAB_WIDEN);
2063 emit_move_insn (loxhi, tem);
2064
2065 emit_label (after_lopart_neg);
2066 }
2067
2068 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2069 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
2070 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
2071 1, OPTAB_WIDEN);
2072 emit_move_insn (loxhi, tem);
2073
2074 /* if (loxhi >> (bitsize / 2)
2075 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2076 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2077 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
2078 NULL_RTX, 0);
2079 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
2080 rtx signbitloxhi = const0_rtx;
2081 if (!uns)
2082 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
2083 convert_modes (hmode, mode,
2084 loxhi, 0),
2085 hprec - 1, NULL_RTX, 0);
2086
2087 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
2088 NULL_RTX, NULL, do_overflow,
2089 profile_probability::very_unlikely ());
2090
2091 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2092 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
2093 NULL_RTX, 1);
2094 tem = convert_modes (mode, hmode,
2095 convert_modes (hmode, mode, lo0xlo1, 1), 1);
2096
2097 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
2098 1, OPTAB_WIDEN);
2099 if (tem != res)
2100 emit_move_insn (res, tem);
2101 emit_jump (done_label);
2102
2103 emit_label (both_ops_large);
2104
2105 /* If both operands are large (not sign (!uns) or zero (uns)
2106 extended from hmode), then perform the full multiplication
2107 which will be the result of the operation.
2108 The only cases which don't overflow are for signed multiplication
2109 some cases where both hipart0 and highpart1 are 0 or -1.
2110 For unsigned multiplication when high parts are both non-zero
2111 this overflows always. */
2112 ops.code = MULT_EXPR;
2113 ops.op0 = make_tree (type, op0);
2114 ops.op1 = make_tree (type, op1);
2115 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2116 emit_move_insn (res, tem);
2117
2118 if (!uns)
2119 {
2120 if (!op0_medium_p)
2121 {
2122 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
2123 NULL_RTX, 1, OPTAB_WIDEN);
2124 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2125 NULL_RTX, NULL, do_error,
2126 profile_probability::very_unlikely ());
2127 }
2128
2129 if (!op1_medium_p)
2130 {
2131 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
2132 NULL_RTX, 1, OPTAB_WIDEN);
2133 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2134 NULL_RTX, NULL, do_error,
2135 profile_probability::very_unlikely ());
2136 }
2137
2138 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2139 the same, overflow happened if res is non-positive, if they
2140 are different, overflow happened if res is positive. */
2141 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
2142 emit_jump (hipart_different);
2143 else if (op0_sign == 1 || op1_sign == 1)
2144 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
2145 NULL_RTX, NULL, hipart_different,
2146 profile_probability::even ());
2147
2148 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
2149 NULL_RTX, NULL, do_error,
2150 profile_probability::very_unlikely ());
2151 emit_jump (done_label);
2152
2153 emit_label (hipart_different);
2154
2155 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
2156 NULL_RTX, NULL, do_error,
2157 profile_probability::very_unlikely ());
2158 emit_jump (done_label);
2159 }
2160
2161 emit_label (do_overflow);
2162
2163 /* Overflow, do full multiplication and fallthru into do_error. */
2164 ops.op0 = make_tree (type, op0);
2165 ops.op1 = make_tree (type, op1);
2166 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2167 emit_move_insn (res, tem);
2168 }
2169 else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
2170 && targetm.scalar_mode_supported_p (wmode))
2171 /* Even emitting a libcall is better than not detecting overflow
2172 at all. */
2173 goto twoxwider;
2174 else
2175 {
2176 gcc_assert (!is_ubsan);
2177 ops.code = MULT_EXPR;
2178 ops.type = type;
2179 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2180 emit_jump (done_label);
2181 }
2182 }
2183
2184 do_error_label:
2185 emit_label (do_error);
2186 if (is_ubsan)
2187 {
2188 /* Expand the ubsan builtin call. */
2189 push_temp_slots ();
2190 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
2191 arg0, arg1, datap);
2192 expand_normal (fn);
2193 pop_temp_slots ();
2194 do_pending_stack_adjust ();
2195 }
2196 else if (lhs)
2197 expand_arith_set_overflow (lhs, target);
2198
2199 /* We're done. */
2200 emit_label (done_label);
2201
2202 /* u1 * u2 -> sr */
2203 if (uns0_p && uns1_p && !unsr_p)
2204 {
2205 rtx_code_label *all_done_label = gen_label_rtx ();
2206 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
2207 NULL, all_done_label, profile_probability::very_likely ());
2208 expand_arith_set_overflow (lhs, target);
2209 emit_label (all_done_label);
2210 }
2211
2212 /* s1 * u2 -> sr */
2213 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
2214 {
2215 rtx_code_label *all_done_label = gen_label_rtx ();
2216 rtx_code_label *set_noovf = gen_label_rtx ();
2217 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2218 NULL, all_done_label, profile_probability::very_likely ());
2219 expand_arith_set_overflow (lhs, target);
2220 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2221 NULL, set_noovf, profile_probability::very_likely ());
2222 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2223 NULL, all_done_label, profile_probability::very_unlikely ());
2224 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2225 all_done_label, profile_probability::very_unlikely ());
2226 emit_label (set_noovf);
2227 write_complex_part (target, const0_rtx, true);
2228 emit_label (all_done_label);
2229 }
2230
2231 if (lhs)
2232 {
2233 if (is_ubsan)
2234 expand_ubsan_result_store (target, res);
2235 else
2236 expand_arith_overflow_result_store (lhs, target, mode, res);
2237 }
2238 }
2239
2240 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2241
2242 static void
2243 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2244 tree arg0, tree arg1)
2245 {
2246 poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2247 rtx_code_label *loop_lab = NULL;
2248 rtx cntvar = NULL_RTX;
2249 tree cntv = NULL_TREE;
2250 tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2251 tree sz = TYPE_SIZE (eltype);
2252 tree data = NULL_TREE;
2253 tree resv = NULL_TREE;
2254 rtx lhsr = NULL_RTX;
2255 rtx resvr = NULL_RTX;
2256 unsigned HOST_WIDE_INT const_cnt = 0;
2257 bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
2258
2259 if (lhs)
2260 {
2261 optab op;
2262 lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2263 if (!VECTOR_MODE_P (GET_MODE (lhsr))
2264 || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2265 optab_default)) == unknown_optab
2266 || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2267 == CODE_FOR_nothing))
2268 {
2269 if (MEM_P (lhsr))
2270 resv = make_tree (TREE_TYPE (lhs), lhsr);
2271 else
2272 {
2273 resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2274 resv = make_tree (TREE_TYPE (lhs), resvr);
2275 }
2276 }
2277 }
2278 if (use_loop_p)
2279 {
2280 do_pending_stack_adjust ();
2281 loop_lab = gen_label_rtx ();
2282 cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2283 cntv = make_tree (sizetype, cntvar);
2284 emit_move_insn (cntvar, const0_rtx);
2285 emit_label (loop_lab);
2286 }
2287 if (TREE_CODE (arg0) != VECTOR_CST)
2288 {
2289 rtx arg0r = expand_normal (arg0);
2290 arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2291 }
2292 if (TREE_CODE (arg1) != VECTOR_CST)
2293 {
2294 rtx arg1r = expand_normal (arg1);
2295 arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2296 }
2297 for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2298 {
2299 tree op0, op1, res = NULL_TREE;
2300 if (use_loop_p)
2301 {
2302 tree atype = build_array_type_nelts (eltype, cnt);
2303 op0 = uniform_vector_p (arg0);
2304 if (op0 == NULL_TREE)
2305 {
2306 op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2307 op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
2308 NULL_TREE, NULL_TREE);
2309 }
2310 op1 = uniform_vector_p (arg1);
2311 if (op1 == NULL_TREE)
2312 {
2313 op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2314 op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
2315 NULL_TREE, NULL_TREE);
2316 }
2317 if (resv)
2318 {
2319 res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2320 res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
2321 NULL_TREE, NULL_TREE);
2322 }
2323 }
2324 else
2325 {
2326 tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2327 op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2328 op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2329 if (resv)
2330 res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2331 bitpos);
2332 }
2333 switch (code)
2334 {
2335 case PLUS_EXPR:
2336 expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
2337 false, false, false, true, &data);
2338 break;
2339 case MINUS_EXPR:
2340 if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2341 expand_neg_overflow (loc, res, op1, true, &data);
2342 else
2343 expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
2344 false, false, false, true, &data);
2345 break;
2346 case MULT_EXPR:
2347 expand_mul_overflow (loc, res, op0, op1, false, false, false,
2348 true, &data);
2349 break;
2350 default:
2351 gcc_unreachable ();
2352 }
2353 }
2354 if (use_loop_p)
2355 {
2356 struct separate_ops ops;
2357 ops.code = PLUS_EXPR;
2358 ops.type = TREE_TYPE (cntv);
2359 ops.op0 = cntv;
2360 ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2361 ops.op2 = NULL_TREE;
2362 ops.location = loc;
2363 rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2364 EXPAND_NORMAL);
2365 if (ret != cntvar)
2366 emit_move_insn (cntvar, ret);
2367 rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2368 do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2369 TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2370 profile_probability::very_likely ());
2371 }
2372 if (lhs && resv == NULL_TREE)
2373 {
2374 struct separate_ops ops;
2375 ops.code = code;
2376 ops.type = TREE_TYPE (arg0);
2377 ops.op0 = arg0;
2378 ops.op1 = arg1;
2379 ops.op2 = NULL_TREE;
2380 ops.location = loc;
2381 rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2382 EXPAND_NORMAL);
2383 if (ret != lhsr)
2384 emit_move_insn (lhsr, ret);
2385 }
2386 else if (resvr)
2387 emit_move_insn (lhsr, resvr);
2388 }
2389
2390 /* Expand UBSAN_CHECK_ADD call STMT. */
2391
2392 static void
2393 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2394 {
2395 location_t loc = gimple_location (stmt);
2396 tree lhs = gimple_call_lhs (stmt);
2397 tree arg0 = gimple_call_arg (stmt, 0);
2398 tree arg1 = gimple_call_arg (stmt, 1);
2399 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2400 expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
2401 else
2402 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
2403 false, false, false, true, NULL);
2404 }
2405
2406 /* Expand UBSAN_CHECK_SUB call STMT. */
2407
2408 static void
2409 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2410 {
2411 location_t loc = gimple_location (stmt);
2412 tree lhs = gimple_call_lhs (stmt);
2413 tree arg0 = gimple_call_arg (stmt, 0);
2414 tree arg1 = gimple_call_arg (stmt, 1);
2415 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2416 expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
2417 else if (integer_zerop (arg0))
2418 expand_neg_overflow (loc, lhs, arg1, true, NULL);
2419 else
2420 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
2421 false, false, false, true, NULL);
2422 }
2423
2424 /* Expand UBSAN_CHECK_MUL call STMT. */
2425
2426 static void
2427 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2428 {
2429 location_t loc = gimple_location (stmt);
2430 tree lhs = gimple_call_lhs (stmt);
2431 tree arg0 = gimple_call_arg (stmt, 0);
2432 tree arg1 = gimple_call_arg (stmt, 1);
2433 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2434 expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
2435 else
2436 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
2437 NULL);
2438 }
2439
2440 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2441
2442 static void
2443 expand_arith_overflow (enum tree_code code, gimple *stmt)
2444 {
2445 tree lhs = gimple_call_lhs (stmt);
2446 if (lhs == NULL_TREE)
2447 return;
2448 tree arg0 = gimple_call_arg (stmt, 0);
2449 tree arg1 = gimple_call_arg (stmt, 1);
2450 tree type = TREE_TYPE (TREE_TYPE (lhs));
2451 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2452 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2453 int unsr_p = TYPE_UNSIGNED (type);
2454 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2455 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2456 int precres = TYPE_PRECISION (type);
2457 location_t loc = gimple_location (stmt);
2458 if (!uns0_p && get_range_pos_neg (arg0) == 1)
2459 uns0_p = true;
2460 if (!uns1_p && get_range_pos_neg (arg1) == 1)
2461 uns1_p = true;
2462 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
2463 prec0 = MIN (prec0, pr);
2464 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
2465 prec1 = MIN (prec1, pr);
2466
2467 /* If uns0_p && uns1_p, precop is minimum needed precision
2468 of unsigned type to hold the exact result, otherwise
2469 precop is minimum needed precision of signed type to
2470 hold the exact result. */
2471 int precop;
2472 if (code == MULT_EXPR)
2473 precop = prec0 + prec1 + (uns0_p != uns1_p);
2474 else
2475 {
2476 if (uns0_p == uns1_p)
2477 precop = MAX (prec0, prec1) + 1;
2478 else if (uns0_p)
2479 precop = MAX (prec0 + 1, prec1) + 1;
2480 else
2481 precop = MAX (prec0, prec1 + 1) + 1;
2482 }
2483 int orig_precres = precres;
2484
2485 do
2486 {
2487 if ((uns0_p && uns1_p)
2488 ? ((precop + !unsr_p) <= precres
2489 /* u1 - u2 -> ur can overflow, no matter what precision
2490 the result has. */
2491 && (code != MINUS_EXPR || !unsr_p))
2492 : (!unsr_p && precop <= precres))
2493 {
2494 /* The infinity precision result will always fit into result. */
2495 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2496 write_complex_part (target, const0_rtx, true);
2497 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2498 struct separate_ops ops;
2499 ops.code = code;
2500 ops.type = type;
2501 ops.op0 = fold_convert_loc (loc, type, arg0);
2502 ops.op1 = fold_convert_loc (loc, type, arg1);
2503 ops.op2 = NULL_TREE;
2504 ops.location = loc;
2505 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2506 expand_arith_overflow_result_store (lhs, target, mode, tem);
2507 return;
2508 }
2509
2510 /* For operations with low precision, if target doesn't have them, start
2511 with precres widening right away, otherwise do it only if the most
2512 simple cases can't be used. */
2513 const int min_precision = targetm.min_arithmetic_precision ();
2514 if (orig_precres == precres && precres < min_precision)
2515 ;
2516 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2517 && prec1 <= precres)
2518 || ((!uns0_p || !uns1_p) && !unsr_p
2519 && prec0 + uns0_p <= precres
2520 && prec1 + uns1_p <= precres))
2521 {
2522 arg0 = fold_convert_loc (loc, type, arg0);
2523 arg1 = fold_convert_loc (loc, type, arg1);
2524 switch (code)
2525 {
2526 case MINUS_EXPR:
2527 if (integer_zerop (arg0) && !unsr_p)
2528 {
2529 expand_neg_overflow (loc, lhs, arg1, false, NULL);
2530 return;
2531 }
2532 /* FALLTHRU */
2533 case PLUS_EXPR:
2534 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2535 unsr_p, unsr_p, false, NULL);
2536 return;
2537 case MULT_EXPR:
2538 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2539 unsr_p, unsr_p, false, NULL);
2540 return;
2541 default:
2542 gcc_unreachable ();
2543 }
2544 }
2545
2546 /* For sub-word operations, retry with a wider type first. */
2547 if (orig_precres == precres && precop <= BITS_PER_WORD)
2548 {
2549 int p = MAX (min_precision, precop);
2550 scalar_int_mode m = smallest_int_mode_for_size (p);
2551 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2552 uns0_p && uns1_p
2553 && unsr_p);
2554 p = TYPE_PRECISION (optype);
2555 if (p > precres)
2556 {
2557 precres = p;
2558 unsr_p = TYPE_UNSIGNED (optype);
2559 type = optype;
2560 continue;
2561 }
2562 }
2563
2564 if (prec0 <= precres && prec1 <= precres)
2565 {
2566 tree types[2];
2567 if (unsr_p)
2568 {
2569 types[0] = build_nonstandard_integer_type (precres, 0);
2570 types[1] = type;
2571 }
2572 else
2573 {
2574 types[0] = type;
2575 types[1] = build_nonstandard_integer_type (precres, 1);
2576 }
2577 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2578 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2579 if (code != MULT_EXPR)
2580 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2581 uns0_p, uns1_p, false, NULL);
2582 else
2583 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2584 uns0_p, uns1_p, false, NULL);
2585 return;
2586 }
2587
2588 /* Retry with a wider type. */
2589 if (orig_precres == precres)
2590 {
2591 int p = MAX (prec0, prec1);
2592 scalar_int_mode m = smallest_int_mode_for_size (p);
2593 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2594 uns0_p && uns1_p
2595 && unsr_p);
2596 p = TYPE_PRECISION (optype);
2597 if (p > precres)
2598 {
2599 precres = p;
2600 unsr_p = TYPE_UNSIGNED (optype);
2601 type = optype;
2602 continue;
2603 }
2604 }
2605
2606 gcc_unreachable ();
2607 }
2608 while (1);
2609 }
2610
2611 /* Expand ADD_OVERFLOW STMT. */
2612
2613 static void
2614 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2615 {
2616 expand_arith_overflow (PLUS_EXPR, stmt);
2617 }
2618
2619 /* Expand SUB_OVERFLOW STMT. */
2620
2621 static void
2622 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2623 {
2624 expand_arith_overflow (MINUS_EXPR, stmt);
2625 }
2626
2627 /* Expand MUL_OVERFLOW STMT. */
2628
2629 static void
2630 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2631 {
2632 expand_arith_overflow (MULT_EXPR, stmt);
2633 }
2634
2635 /* This should get folded in tree-vectorizer.cc. */
2636
2637 static void
2638 expand_LOOP_VECTORIZED (internal_fn, gcall *)
2639 {
2640 gcc_unreachable ();
2641 }
2642
2643 /* This should get folded in tree-vectorizer.cc. */
2644
2645 static void
2646 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
2647 {
2648 gcc_unreachable ();
2649 }
2650
2651 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2652 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2653
2654 static tree
2655 expand_call_mem_ref (tree type, gcall *stmt, int index)
2656 {
2657 tree addr = gimple_call_arg (stmt, index);
2658 tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
2659 unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
2660 if (TYPE_ALIGN (type) != align)
2661 type = build_aligned_type (type, align);
2662
2663 tree tmp = addr;
2664 if (TREE_CODE (tmp) == SSA_NAME)
2665 {
2666 gimple *def = SSA_NAME_DEF_STMT (tmp);
2667 if (gimple_assign_single_p (def))
2668 tmp = gimple_assign_rhs1 (def);
2669 }
2670
2671 if (TREE_CODE (tmp) == ADDR_EXPR)
2672 {
2673 tree mem = TREE_OPERAND (tmp, 0);
2674 if (TREE_CODE (mem) == TARGET_MEM_REF
2675 && types_compatible_p (TREE_TYPE (mem), type))
2676 {
2677 tree offset = TMR_OFFSET (mem);
2678 if (type != TREE_TYPE (mem)
2679 || alias_ptr_type != TREE_TYPE (offset)
2680 || !integer_zerop (offset))
2681 {
2682 mem = copy_node (mem);
2683 TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
2684 wi::to_poly_wide (offset));
2685 TREE_TYPE (mem) = type;
2686 }
2687 return mem;
2688 }
2689 }
2690
2691 return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
2692 }
2693
2694 /* Expand MASK_LOAD{,_LANES} or LEN_LOAD call STMT using optab OPTAB. */
2695
2696 static void
2697 expand_partial_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2698 {
2699 class expand_operand ops[4];
2700 tree type, lhs, rhs, maskt, biast;
2701 rtx mem, target, mask, bias;
2702 insn_code icode;
2703
2704 maskt = gimple_call_arg (stmt, 2);
2705 lhs = gimple_call_lhs (stmt);
2706 if (lhs == NULL_TREE)
2707 return;
2708 type = TREE_TYPE (lhs);
2709 rhs = expand_call_mem_ref (type, stmt, 0);
2710
2711 if (optab == vec_mask_load_lanes_optab)
2712 icode = get_multi_vector_move (type, optab);
2713 else if (optab == len_load_optab)
2714 icode = direct_optab_handler (optab, TYPE_MODE (type));
2715 else
2716 icode = convert_optab_handler (optab, TYPE_MODE (type),
2717 TYPE_MODE (TREE_TYPE (maskt)));
2718
2719 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2720 gcc_assert (MEM_P (mem));
2721 mask = expand_normal (maskt);
2722 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2723 create_output_operand (&ops[0], target, TYPE_MODE (type));
2724 create_fixed_operand (&ops[1], mem);
2725 if (optab == len_load_optab)
2726 {
2727 create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
2728 TYPE_UNSIGNED (TREE_TYPE (maskt)));
2729 biast = gimple_call_arg (stmt, 3);
2730 bias = expand_normal (biast);
2731 create_input_operand (&ops[3], bias, QImode);
2732 expand_insn (icode, 4, ops);
2733 }
2734 else
2735 {
2736 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2737 expand_insn (icode, 3, ops);
2738 }
2739
2740 if (!rtx_equal_p (target, ops[0].value))
2741 emit_move_insn (target, ops[0].value);
2742 }
2743
2744 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2745 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2746 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2747
2748 /* Expand MASK_STORE{,_LANES} or LEN_STORE call STMT using optab OPTAB. */
2749
2750 static void
2751 expand_partial_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2752 {
2753 class expand_operand ops[4];
2754 tree type, lhs, rhs, maskt, biast;
2755 rtx mem, reg, mask, bias;
2756 insn_code icode;
2757
2758 maskt = gimple_call_arg (stmt, 2);
2759 rhs = gimple_call_arg (stmt, 3);
2760 type = TREE_TYPE (rhs);
2761 lhs = expand_call_mem_ref (type, stmt, 0);
2762
2763 if (optab == vec_mask_store_lanes_optab)
2764 icode = get_multi_vector_move (type, optab);
2765 else if (optab == len_store_optab)
2766 icode = direct_optab_handler (optab, TYPE_MODE (type));
2767 else
2768 icode = convert_optab_handler (optab, TYPE_MODE (type),
2769 TYPE_MODE (TREE_TYPE (maskt)));
2770
2771 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2772 gcc_assert (MEM_P (mem));
2773 mask = expand_normal (maskt);
2774 reg = expand_normal (rhs);
2775 create_fixed_operand (&ops[0], mem);
2776 create_input_operand (&ops[1], reg, TYPE_MODE (type));
2777 if (optab == len_store_optab)
2778 {
2779 create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
2780 TYPE_UNSIGNED (TREE_TYPE (maskt)));
2781 biast = gimple_call_arg (stmt, 4);
2782 bias = expand_normal (biast);
2783 create_input_operand (&ops[3], bias, QImode);
2784 expand_insn (icode, 4, ops);
2785 }
2786 else
2787 {
2788 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2789 expand_insn (icode, 3, ops);
2790 }
2791 }
2792
2793 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
2794 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2795 #define expand_len_store_optab_fn expand_partial_store_optab_fn
2796
2797 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
2798 The expansion of STMT happens based on OPTAB table associated. */
2799
2800 static void
2801 expand_vec_cond_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2802 {
2803 class expand_operand ops[6];
2804 insn_code icode;
2805 tree lhs = gimple_call_lhs (stmt);
2806 tree op0a = gimple_call_arg (stmt, 0);
2807 tree op0b = gimple_call_arg (stmt, 1);
2808 tree op1 = gimple_call_arg (stmt, 2);
2809 tree op2 = gimple_call_arg (stmt, 3);
2810 enum tree_code tcode = (tree_code) int_cst_value (gimple_call_arg (stmt, 4));
2811
2812 tree vec_cond_type = TREE_TYPE (lhs);
2813 tree op_mode = TREE_TYPE (op0a);
2814 bool unsignedp = TYPE_UNSIGNED (op_mode);
2815
2816 machine_mode mode = TYPE_MODE (vec_cond_type);
2817 machine_mode cmp_op_mode = TYPE_MODE (op_mode);
2818
2819 icode = convert_optab_handler (optab, mode, cmp_op_mode);
2820 rtx comparison
2821 = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp, icode, 4);
2822 rtx rtx_op1 = expand_normal (op1);
2823 rtx rtx_op2 = expand_normal (op2);
2824
2825 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2826 create_output_operand (&ops[0], target, mode);
2827 create_input_operand (&ops[1], rtx_op1, mode);
2828 create_input_operand (&ops[2], rtx_op2, mode);
2829 create_fixed_operand (&ops[3], comparison);
2830 create_fixed_operand (&ops[4], XEXP (comparison, 0));
2831 create_fixed_operand (&ops[5], XEXP (comparison, 1));
2832 expand_insn (icode, 6, ops);
2833 if (!rtx_equal_p (ops[0].value, target))
2834 emit_move_insn (target, ops[0].value);
2835 }
2836
2837 /* Expand VCOND_MASK optab internal function.
2838 The expansion of STMT happens based on OPTAB table associated. */
2839
2840 static void
2841 expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2842 {
2843 class expand_operand ops[4];
2844
2845 tree lhs = gimple_call_lhs (stmt);
2846 tree op0 = gimple_call_arg (stmt, 0);
2847 tree op1 = gimple_call_arg (stmt, 1);
2848 tree op2 = gimple_call_arg (stmt, 2);
2849 tree vec_cond_type = TREE_TYPE (lhs);
2850
2851 machine_mode mode = TYPE_MODE (vec_cond_type);
2852 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
2853 enum insn_code icode = convert_optab_handler (optab, mode, mask_mode);
2854 rtx mask, rtx_op1, rtx_op2;
2855
2856 gcc_assert (icode != CODE_FOR_nothing);
2857
2858 mask = expand_normal (op0);
2859 rtx_op1 = expand_normal (op1);
2860 rtx_op2 = expand_normal (op2);
2861
2862 mask = force_reg (mask_mode, mask);
2863 rtx_op1 = force_reg (mode, rtx_op1);
2864
2865 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2866 create_output_operand (&ops[0], target, mode);
2867 create_input_operand (&ops[1], rtx_op1, mode);
2868 create_input_operand (&ops[2], rtx_op2, mode);
2869 create_input_operand (&ops[3], mask, mask_mode);
2870 expand_insn (icode, 4, ops);
2871 if (!rtx_equal_p (ops[0].value, target))
2872 emit_move_insn (target, ops[0].value);
2873 }
2874
2875 /* Expand VEC_SET internal functions. */
2876
2877 static void
2878 expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2879 {
2880 tree lhs = gimple_call_lhs (stmt);
2881 tree op0 = gimple_call_arg (stmt, 0);
2882 tree op1 = gimple_call_arg (stmt, 1);
2883 tree op2 = gimple_call_arg (stmt, 2);
2884 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2885 rtx src = expand_normal (op0);
2886
2887 machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
2888 scalar_mode innermode = GET_MODE_INNER (outermode);
2889
2890 rtx value = expand_normal (op1);
2891 rtx pos = expand_normal (op2);
2892
2893 class expand_operand ops[3];
2894 enum insn_code icode = optab_handler (optab, outermode);
2895
2896 if (icode != CODE_FOR_nothing)
2897 {
2898 rtx temp = gen_reg_rtx (outermode);
2899 emit_move_insn (temp, src);
2900
2901 create_fixed_operand (&ops[0], temp);
2902 create_input_operand (&ops[1], value, innermode);
2903 create_convert_operand_from (&ops[2], pos, TYPE_MODE (TREE_TYPE (op2)),
2904 true);
2905 if (maybe_expand_insn (icode, 3, ops))
2906 {
2907 emit_move_insn (target, temp);
2908 return;
2909 }
2910 }
2911 gcc_unreachable ();
2912 }
2913
2914 static void
2915 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
2916 {
2917 }
2918
2919 static void
2920 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
2921 {
2922 /* When guessing was done, the hints should be already stripped away. */
2923 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
2924
2925 rtx target;
2926 tree lhs = gimple_call_lhs (stmt);
2927 if (lhs)
2928 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2929 else
2930 target = const0_rtx;
2931 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
2932 if (lhs && val != target)
2933 emit_move_insn (target, val);
2934 }
2935
2936 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2937 should never be called. */
2938
2939 static void
2940 expand_VA_ARG (internal_fn, gcall *)
2941 {
2942 gcc_unreachable ();
2943 }
2944
2945 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2946 dummy function should never be called. */
2947
2948 static void
2949 expand_VEC_CONVERT (internal_fn, gcall *)
2950 {
2951 gcc_unreachable ();
2952 }
2953
2954 /* Expand IFN_RAWMEMCHAR internal function. */
2955
2956 void
2957 expand_RAWMEMCHR (internal_fn, gcall *stmt)
2958 {
2959 expand_operand ops[3];
2960
2961 tree lhs = gimple_call_lhs (stmt);
2962 if (!lhs)
2963 return;
2964 machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
2965 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2966 create_output_operand (&ops[0], lhs_rtx, lhs_mode);
2967
2968 tree mem = gimple_call_arg (stmt, 0);
2969 rtx mem_rtx = get_memory_rtx (mem, NULL);
2970 create_fixed_operand (&ops[1], mem_rtx);
2971
2972 tree pattern = gimple_call_arg (stmt, 1);
2973 machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
2974 rtx pattern_rtx = expand_normal (pattern);
2975 create_input_operand (&ops[2], pattern_rtx, mode);
2976
2977 insn_code icode = direct_optab_handler (rawmemchr_optab, mode);
2978
2979 expand_insn (icode, 3, ops);
2980 if (!rtx_equal_p (lhs_rtx, ops[0].value))
2981 emit_move_insn (lhs_rtx, ops[0].value);
2982 }
2983
2984 /* Expand the IFN_UNIQUE function according to its first argument. */
2985
2986 static void
2987 expand_UNIQUE (internal_fn, gcall *stmt)
2988 {
2989 rtx pattern = NULL_RTX;
2990 enum ifn_unique_kind kind
2991 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
2992
2993 switch (kind)
2994 {
2995 default:
2996 gcc_unreachable ();
2997
2998 case IFN_UNIQUE_UNSPEC:
2999 if (targetm.have_unique ())
3000 pattern = targetm.gen_unique ();
3001 break;
3002
3003 case IFN_UNIQUE_OACC_FORK:
3004 case IFN_UNIQUE_OACC_JOIN:
3005 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
3006 {
3007 tree lhs = gimple_call_lhs (stmt);
3008 rtx target = const0_rtx;
3009
3010 if (lhs)
3011 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3012
3013 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
3014 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
3015
3016 if (kind == IFN_UNIQUE_OACC_FORK)
3017 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
3018 else
3019 pattern = targetm.gen_oacc_join (target, data_dep, axis);
3020 }
3021 else
3022 gcc_unreachable ();
3023 break;
3024 }
3025
3026 if (pattern)
3027 emit_insn (pattern);
3028 }
3029
3030 /* Expand the IFN_DEFERRED_INIT function:
3031 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3032
3033 Initialize the LHS with zero/pattern according to its second argument
3034 INIT_TYPE:
3035 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3036 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3037 to initialize;
3038 The LHS variable is initialized including paddings.
3039 The reasons to choose 0xFE for pattern initialization are:
3040 1. It is a non-canonical virtual address on x86_64, and at the
3041 high end of the i386 kernel address space.
3042 2. It is a very large float value (-1.694739530317379e+38).
3043 3. It is also an unusual number for integers. */
3044 #define INIT_PATTERN_VALUE 0xFE
3045 static void
3046 expand_DEFERRED_INIT (internal_fn, gcall *stmt)
3047 {
3048 tree lhs = gimple_call_lhs (stmt);
3049 tree var_size = gimple_call_arg (stmt, 0);
3050 enum auto_init_type init_type
3051 = (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
3052 bool reg_lhs = true;
3053
3054 tree var_type = TREE_TYPE (lhs);
3055 gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
3056
3057 if (TREE_CODE (lhs) == SSA_NAME)
3058 reg_lhs = true;
3059 else
3060 {
3061 tree lhs_base = lhs;
3062 while (handled_component_p (lhs_base))
3063 lhs_base = TREE_OPERAND (lhs_base, 0);
3064 reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
3065 || non_mem_decl_p (lhs_base));
3066 /* If this expands to a register and the underlying decl is wrapped in
3067 a MEM_REF that just serves as an access type change expose the decl
3068 if it is of correct size. This avoids a situation as in PR103271
3069 if the target does not support a direct move to the registers mode. */
3070 if (reg_lhs
3071 && TREE_CODE (lhs_base) == MEM_REF
3072 && TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
3073 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
3074 && integer_zerop (TREE_OPERAND (lhs_base, 1))
3075 && tree_fits_uhwi_p (var_size)
3076 && tree_int_cst_equal
3077 (var_size,
3078 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
3079 {
3080 lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
3081 var_type = TREE_TYPE (lhs);
3082 }
3083 }
3084
3085 if (!reg_lhs)
3086 {
3087 /* If the variable is not in register, expand to a memset
3088 to initialize it. */
3089 mark_addressable (lhs);
3090 tree var_addr = build_fold_addr_expr (lhs);
3091
3092 tree value = (init_type == AUTO_INIT_PATTERN)
3093 ? build_int_cst (integer_type_node,
3094 INIT_PATTERN_VALUE)
3095 : integer_zero_node;
3096 tree m_call = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET),
3097 3, var_addr, value, var_size);
3098 /* Expand this memset call. */
3099 expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
3100 }
3101 else
3102 {
3103 /* If this variable is in a register use expand_assignment.
3104 For boolean scalars force zero-init. */
3105 tree init;
3106 scalar_int_mode var_mode;
3107 if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
3108 && tree_fits_uhwi_p (var_size)
3109 && (init_type == AUTO_INIT_PATTERN
3110 || !is_gimple_reg_type (var_type))
3111 && int_mode_for_size (tree_to_uhwi (var_size) * BITS_PER_UNIT,
3112 0).exists (&var_mode)
3113 && have_insn_for (SET, var_mode))
3114 {
3115 unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
3116 unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
3117 memset (buf, (init_type == AUTO_INIT_PATTERN
3118 ? INIT_PATTERN_VALUE : 0), total_bytes);
3119 tree itype = build_nonstandard_integer_type
3120 (total_bytes * BITS_PER_UNIT, 1);
3121 wide_int w = wi::from_buffer (buf, total_bytes);
3122 init = wide_int_to_tree (itype, w);
3123 /* Pun the LHS to make sure its type has constant size
3124 unless it is an SSA name where that's already known. */
3125 if (TREE_CODE (lhs) != SSA_NAME)
3126 lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
3127 else
3128 init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
3129 }
3130 else
3131 /* Use zero-init also for variable-length sizes. */
3132 init = build_zero_cst (var_type);
3133
3134 expand_assignment (lhs, init, false);
3135 }
3136 }
3137
3138 /* The size of an OpenACC compute dimension. */
3139
3140 static void
3141 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
3142 {
3143 tree lhs = gimple_call_lhs (stmt);
3144
3145 if (!lhs)
3146 return;
3147
3148 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3149 if (targetm.have_oacc_dim_size ())
3150 {
3151 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3152 VOIDmode, EXPAND_NORMAL);
3153 emit_insn (targetm.gen_oacc_dim_size (target, dim));
3154 }
3155 else
3156 emit_move_insn (target, GEN_INT (1));
3157 }
3158
3159 /* The position of an OpenACC execution engine along one compute axis. */
3160
3161 static void
3162 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
3163 {
3164 tree lhs = gimple_call_lhs (stmt);
3165
3166 if (!lhs)
3167 return;
3168
3169 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3170 if (targetm.have_oacc_dim_pos ())
3171 {
3172 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3173 VOIDmode, EXPAND_NORMAL);
3174 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
3175 }
3176 else
3177 emit_move_insn (target, const0_rtx);
3178 }
3179
3180 /* This is expanded by oacc_device_lower pass. */
3181
3182 static void
3183 expand_GOACC_LOOP (internal_fn, gcall *)
3184 {
3185 gcc_unreachable ();
3186 }
3187
3188 /* This is expanded by oacc_device_lower pass. */
3189
3190 static void
3191 expand_GOACC_REDUCTION (internal_fn, gcall *)
3192 {
3193 gcc_unreachable ();
3194 }
3195
3196 /* This is expanded by oacc_device_lower pass. */
3197
3198 static void
3199 expand_GOACC_TILE (internal_fn, gcall *)
3200 {
3201 gcc_unreachable ();
3202 }
3203
3204 /* Set errno to EDOM. */
3205
3206 static void
3207 expand_SET_EDOM (internal_fn, gcall *)
3208 {
3209 #ifdef TARGET_EDOM
3210 #ifdef GEN_ERRNO_RTX
3211 rtx errno_rtx = GEN_ERRNO_RTX;
3212 #else
3213 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
3214 #endif
3215 emit_move_insn (errno_rtx,
3216 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
3217 #else
3218 gcc_unreachable ();
3219 #endif
3220 }
3221
3222 /* Expand atomic bit test and set. */
3223
3224 static void
3225 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
3226 {
3227 expand_ifn_atomic_bit_test_and (call);
3228 }
3229
3230 /* Expand atomic bit test and complement. */
3231
3232 static void
3233 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
3234 {
3235 expand_ifn_atomic_bit_test_and (call);
3236 }
3237
3238 /* Expand atomic bit test and reset. */
3239
3240 static void
3241 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
3242 {
3243 expand_ifn_atomic_bit_test_and (call);
3244 }
3245
3246 /* Expand atomic bit test and set. */
3247
3248 static void
3249 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
3250 {
3251 expand_ifn_atomic_compare_exchange (call);
3252 }
3253
3254 /* Expand atomic add fetch and cmp with 0. */
3255
3256 static void
3257 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
3258 {
3259 expand_ifn_atomic_op_fetch_cmp_0 (call);
3260 }
3261
3262 /* Expand atomic sub fetch and cmp with 0. */
3263
3264 static void
3265 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
3266 {
3267 expand_ifn_atomic_op_fetch_cmp_0 (call);
3268 }
3269
3270 /* Expand atomic and fetch and cmp with 0. */
3271
3272 static void
3273 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
3274 {
3275 expand_ifn_atomic_op_fetch_cmp_0 (call);
3276 }
3277
3278 /* Expand atomic or fetch and cmp with 0. */
3279
3280 static void
3281 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
3282 {
3283 expand_ifn_atomic_op_fetch_cmp_0 (call);
3284 }
3285
3286 /* Expand atomic xor fetch and cmp with 0. */
3287
3288 static void
3289 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
3290 {
3291 expand_ifn_atomic_op_fetch_cmp_0 (call);
3292 }
3293
3294 /* Expand LAUNDER to assignment, lhs = arg0. */
3295
3296 static void
3297 expand_LAUNDER (internal_fn, gcall *call)
3298 {
3299 tree lhs = gimple_call_lhs (call);
3300
3301 if (!lhs)
3302 return;
3303
3304 expand_assignment (lhs, gimple_call_arg (call, 0), false);
3305 }
3306
3307 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3308
3309 static void
3310 expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3311 {
3312 internal_fn ifn = gimple_call_internal_fn (stmt);
3313 int rhs_index = internal_fn_stored_value_index (ifn);
3314 int mask_index = internal_fn_mask_index (ifn);
3315 tree base = gimple_call_arg (stmt, 0);
3316 tree offset = gimple_call_arg (stmt, 1);
3317 tree scale = gimple_call_arg (stmt, 2);
3318 tree rhs = gimple_call_arg (stmt, rhs_index);
3319
3320 rtx base_rtx = expand_normal (base);
3321 rtx offset_rtx = expand_normal (offset);
3322 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3323 rtx rhs_rtx = expand_normal (rhs);
3324
3325 class expand_operand ops[6];
3326 int i = 0;
3327 create_address_operand (&ops[i++], base_rtx);
3328 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3329 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3330 create_integer_operand (&ops[i++], scale_int);
3331 create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
3332 if (mask_index >= 0)
3333 {
3334 tree mask = gimple_call_arg (stmt, mask_index);
3335 rtx mask_rtx = expand_normal (mask);
3336 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
3337 }
3338
3339 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
3340 TYPE_MODE (TREE_TYPE (offset)));
3341 expand_insn (icode, i, ops);
3342 }
3343
3344 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3345
3346 static void
3347 expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3348 {
3349 tree lhs = gimple_call_lhs (stmt);
3350 tree base = gimple_call_arg (stmt, 0);
3351 tree offset = gimple_call_arg (stmt, 1);
3352 tree scale = gimple_call_arg (stmt, 2);
3353
3354 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3355 rtx base_rtx = expand_normal (base);
3356 rtx offset_rtx = expand_normal (offset);
3357 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3358
3359 int i = 0;
3360 class expand_operand ops[6];
3361 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
3362 create_address_operand (&ops[i++], base_rtx);
3363 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3364 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3365 create_integer_operand (&ops[i++], scale_int);
3366 if (optab == mask_gather_load_optab)
3367 {
3368 tree mask = gimple_call_arg (stmt, 4);
3369 rtx mask_rtx = expand_normal (mask);
3370 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
3371 }
3372 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
3373 TYPE_MODE (TREE_TYPE (offset)));
3374 expand_insn (icode, i, ops);
3375 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3376 emit_move_insn (lhs_rtx, ops[0].value);
3377 }
3378
3379 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3380 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3381
3382 static bool
3383 contains_call_div_mod (rtx_insn *insn)
3384 {
3385 subrtx_iterator::array_type array;
3386 for (; insn; insn = NEXT_INSN (insn))
3387 if (CALL_P (insn))
3388 return true;
3389 else if (INSN_P (insn))
3390 FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
3391 switch (GET_CODE (*iter))
3392 {
3393 case CALL:
3394 case DIV:
3395 case UDIV:
3396 case MOD:
3397 case UMOD:
3398 return true;
3399 default:
3400 break;
3401 }
3402 return false;
3403 }
3404
3405 /* Expand DIVMOD() using:
3406 a) optab handler for udivmod/sdivmod if it is available.
3407 b) If optab_handler doesn't exist, generate call to
3408 target-specific divmod libfunc. */
3409
3410 static void
3411 expand_DIVMOD (internal_fn, gcall *call_stmt)
3412 {
3413 tree lhs = gimple_call_lhs (call_stmt);
3414 tree arg0 = gimple_call_arg (call_stmt, 0);
3415 tree arg1 = gimple_call_arg (call_stmt, 1);
3416
3417 gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
3418 tree type = TREE_TYPE (TREE_TYPE (lhs));
3419 machine_mode mode = TYPE_MODE (type);
3420 bool unsignedp = TYPE_UNSIGNED (type);
3421 optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
3422
3423 rtx op0 = expand_normal (arg0);
3424 rtx op1 = expand_normal (arg1);
3425 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3426
3427 rtx quotient = NULL_RTX, remainder = NULL_RTX;
3428 rtx_insn *insns = NULL;
3429
3430 if (TREE_CODE (arg1) == INTEGER_CST)
3431 {
3432 /* For DIVMOD by integral constants, there could be efficient code
3433 expanded inline e.g. using shifts and plus/minus. Try to expand
3434 the division and modulo and if it emits any library calls or any
3435 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3436 divmod libcall. */
3437 scalar_int_mode int_mode;
3438 if (remainder == NULL_RTX
3439 && optimize
3440 && CONST_INT_P (op1)
3441 && !pow2p_hwi (INTVAL (op1))
3442 && is_int_mode (TYPE_MODE (type), &int_mode)
3443 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3444 && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
3445 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
3446 && optimize_insn_for_speed_p ())
3447 {
3448 rtx_insn *last = get_last_insn ();
3449 remainder = NULL_RTX;
3450 quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
3451 TYPE_UNSIGNED (type));
3452 if (quotient != NULL_RTX)
3453 {
3454 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
3455 {
3456 rtx_insn *move = emit_move_insn (quotient, quotient);
3457 set_dst_reg_note (move, REG_EQUAL,
3458 gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3459 ? UDIV : DIV, int_mode,
3460 copy_rtx (op0), op1),
3461 quotient);
3462 move = emit_move_insn (remainder, remainder);
3463 set_dst_reg_note (move, REG_EQUAL,
3464 gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3465 ? UMOD : MOD, int_mode,
3466 copy_rtx (op0), op1),
3467 quotient);
3468 }
3469 }
3470 else
3471 delete_insns_since (last);
3472 }
3473
3474 if (remainder == NULL_RTX)
3475 {
3476 struct separate_ops ops;
3477 ops.code = TRUNC_DIV_EXPR;
3478 ops.type = type;
3479 ops.op0 = make_tree (ops.type, op0);
3480 ops.op1 = arg1;
3481 ops.op2 = NULL_TREE;
3482 ops.location = gimple_location (call_stmt);
3483 start_sequence ();
3484 quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
3485 if (contains_call_div_mod (get_insns ()))
3486 quotient = NULL_RTX;
3487 else
3488 {
3489 ops.code = TRUNC_MOD_EXPR;
3490 remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
3491 EXPAND_NORMAL);
3492 if (contains_call_div_mod (get_insns ()))
3493 remainder = NULL_RTX;
3494 }
3495 if (remainder)
3496 insns = get_insns ();
3497 end_sequence ();
3498 }
3499 }
3500
3501 if (remainder)
3502 emit_insn (insns);
3503
3504 /* Check if optab_handler exists for divmod_optab for given mode. */
3505 else if (optab_handler (tab, mode) != CODE_FOR_nothing)
3506 {
3507 quotient = gen_reg_rtx (mode);
3508 remainder = gen_reg_rtx (mode);
3509 expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
3510 }
3511
3512 /* Generate call to divmod libfunc if it exists. */
3513 else if (rtx libfunc = optab_libfunc (tab, mode))
3514 targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
3515 &quotient, &remainder);
3516
3517 else
3518 gcc_unreachable ();
3519
3520 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3521 expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
3522 make_tree (TREE_TYPE (arg0), quotient),
3523 make_tree (TREE_TYPE (arg1), remainder)),
3524 target, VOIDmode, EXPAND_NORMAL);
3525 }
3526
3527 /* Expand a NOP. */
3528
3529 static void
3530 expand_NOP (internal_fn, gcall *)
3531 {
3532 /* Nothing. But it shouldn't really prevail. */
3533 }
3534
3535 /* Coroutines, all should have been processed at this stage. */
3536
3537 static void
3538 expand_CO_FRAME (internal_fn, gcall *)
3539 {
3540 gcc_unreachable ();
3541 }
3542
3543 static void
3544 expand_CO_YIELD (internal_fn, gcall *)
3545 {
3546 gcc_unreachable ();
3547 }
3548
3549 static void
3550 expand_CO_SUSPN (internal_fn, gcall *)
3551 {
3552 gcc_unreachable ();
3553 }
3554
3555 static void
3556 expand_CO_ACTOR (internal_fn, gcall *)
3557 {
3558 gcc_unreachable ();
3559 }
3560
3561 /* Expand a call to FN using the operands in STMT. FN has a single
3562 output operand and NARGS input operands. */
3563
3564 static void
3565 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
3566 unsigned int nargs)
3567 {
3568 expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
3569
3570 tree_pair types = direct_internal_fn_types (fn, stmt);
3571 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
3572 gcc_assert (icode != CODE_FOR_nothing);
3573
3574 tree lhs = gimple_call_lhs (stmt);
3575 rtx lhs_rtx = NULL_RTX;
3576 if (lhs)
3577 lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3578
3579 /* Do not assign directly to a promoted subreg, since there is no
3580 guarantee that the instruction will leave the upper bits of the
3581 register in the state required by SUBREG_PROMOTED_SIGN. */
3582 rtx dest = lhs_rtx;
3583 if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
3584 dest = NULL_RTX;
3585
3586 create_output_operand (&ops[0], dest, insn_data[icode].operand[0].mode);
3587
3588 for (unsigned int i = 0; i < nargs; ++i)
3589 {
3590 tree rhs = gimple_call_arg (stmt, i);
3591 tree rhs_type = TREE_TYPE (rhs);
3592 rtx rhs_rtx = expand_normal (rhs);
3593 if (INTEGRAL_TYPE_P (rhs_type))
3594 create_convert_operand_from (&ops[i + 1], rhs_rtx,
3595 TYPE_MODE (rhs_type),
3596 TYPE_UNSIGNED (rhs_type));
3597 else
3598 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
3599 }
3600
3601 expand_insn (icode, nargs + 1, ops);
3602 if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
3603 {
3604 /* If the return value has an integral type, convert the instruction
3605 result to that type. This is useful for things that return an
3606 int regardless of the size of the input. If the instruction result
3607 is smaller than required, assume that it is signed.
3608
3609 If the return value has a nonintegral type, its mode must match
3610 the instruction result. */
3611 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
3612 {
3613 /* If this is a scalar in a register that is stored in a wider
3614 mode than the declared mode, compute the result into its
3615 declared mode and then convert to the wider mode. */
3616 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
3617 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
3618 convert_move (SUBREG_REG (lhs_rtx), tmp,
3619 SUBREG_PROMOTED_SIGN (lhs_rtx));
3620 }
3621 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
3622 emit_move_insn (lhs_rtx, ops[0].value);
3623 else
3624 {
3625 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
3626 convert_move (lhs_rtx, ops[0].value, 0);
3627 }
3628 }
3629 }
3630
3631 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3632
3633 static void
3634 expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3635 {
3636 expand_operand ops[3];
3637 tree rhs_type[2];
3638
3639 tree lhs = gimple_call_lhs (stmt);
3640 tree lhs_type = TREE_TYPE (lhs);
3641 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3642 create_output_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
3643
3644 for (unsigned int i = 0; i < 2; ++i)
3645 {
3646 tree rhs = gimple_call_arg (stmt, i);
3647 rhs_type[i] = TREE_TYPE (rhs);
3648 rtx rhs_rtx = expand_normal (rhs);
3649 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
3650 }
3651
3652 insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
3653 TYPE_MODE (lhs_type));
3654
3655 expand_insn (icode, 3, ops);
3656 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3657 emit_move_insn (lhs_rtx, ops[0].value);
3658 }
3659
3660 /* Expanders for optabs that can use expand_direct_optab_fn. */
3661
3662 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3663 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3664
3665 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3666 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3667
3668 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3669 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3670
3671 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3672 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3673
3674 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3675 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3676
3677 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3678 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3679
3680 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3681 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3682
3683 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3684 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3685
3686 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3687 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3688
3689 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3690 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3691
3692 /* RETURN_TYPE and ARGS are a return type and argument list that are
3693 in principle compatible with FN (which satisfies direct_internal_fn_p).
3694 Return the types that should be used to determine whether the
3695 target supports FN. */
3696
3697 tree_pair
3698 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
3699 {
3700 const direct_internal_fn_info &info = direct_internal_fn (fn);
3701 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
3702 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
3703 return tree_pair (type0, type1);
3704 }
3705
3706 /* CALL is a call whose return type and arguments are in principle
3707 compatible with FN (which satisfies direct_internal_fn_p). Return the
3708 types that should be used to determine whether the target supports FN. */
3709
3710 tree_pair
3711 direct_internal_fn_types (internal_fn fn, gcall *call)
3712 {
3713 const direct_internal_fn_info &info = direct_internal_fn (fn);
3714 tree op0 = (info.type0 < 0
3715 ? gimple_call_lhs (call)
3716 : gimple_call_arg (call, info.type0));
3717 tree op1 = (info.type1 < 0
3718 ? gimple_call_lhs (call)
3719 : gimple_call_arg (call, info.type1));
3720 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
3721 }
3722
3723 /* Return true if OPTAB is supported for TYPES (whose modes should be
3724 the same) when the optimization type is OPT_TYPE. Used for simple
3725 direct optabs. */
3726
3727 static bool
3728 direct_optab_supported_p (direct_optab optab, tree_pair types,
3729 optimization_type opt_type)
3730 {
3731 machine_mode mode = TYPE_MODE (types.first);
3732 gcc_checking_assert (mode == TYPE_MODE (types.second));
3733 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
3734 }
3735
3736 /* Return true if OPTAB is supported for TYPES, where the first type
3737 is the destination and the second type is the source. Used for
3738 convert optabs. */
3739
3740 static bool
3741 convert_optab_supported_p (convert_optab optab, tree_pair types,
3742 optimization_type opt_type)
3743 {
3744 return (convert_optab_handler (optab, TYPE_MODE (types.first),
3745 TYPE_MODE (types.second), opt_type)
3746 != CODE_FOR_nothing);
3747 }
3748
3749 /* Return true if load/store lanes optab OPTAB is supported for
3750 array type TYPES.first when the optimization type is OPT_TYPE. */
3751
3752 static bool
3753 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
3754 optimization_type opt_type)
3755 {
3756 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
3757 machine_mode imode = TYPE_MODE (types.first);
3758 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
3759 return (convert_optab_handler (optab, imode, vmode, opt_type)
3760 != CODE_FOR_nothing);
3761 }
3762
3763 #define direct_unary_optab_supported_p direct_optab_supported_p
3764 #define direct_binary_optab_supported_p direct_optab_supported_p
3765 #define direct_ternary_optab_supported_p direct_optab_supported_p
3766 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3767 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3768 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3769 #define direct_mask_load_optab_supported_p convert_optab_supported_p
3770 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3771 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3772 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3773 #define direct_len_load_optab_supported_p direct_optab_supported_p
3774 #define direct_mask_store_optab_supported_p convert_optab_supported_p
3775 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3776 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3777 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
3778 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
3779 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3780 #define direct_len_store_optab_supported_p direct_optab_supported_p
3781 #define direct_while_optab_supported_p convert_optab_supported_p
3782 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3783 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3784 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3785 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3786 #define direct_vec_set_optab_supported_p direct_optab_supported_p
3787
3788 /* Return the optab used by internal function FN. */
3789
3790 static optab
3791 direct_internal_fn_optab (internal_fn fn, tree_pair types)
3792 {
3793 switch (fn)
3794 {
3795 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3796 case IFN_##CODE: break;
3797 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3798 case IFN_##CODE: return OPTAB##_optab;
3799 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3800 UNSIGNED_OPTAB, TYPE) \
3801 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3802 ? UNSIGNED_OPTAB ## _optab \
3803 : SIGNED_OPTAB ## _optab);
3804 #include "internal-fn.def"
3805
3806 case IFN_LAST:
3807 break;
3808 }
3809 gcc_unreachable ();
3810 }
3811
3812 /* Return the optab used by internal function FN. */
3813
3814 static optab
3815 direct_internal_fn_optab (internal_fn fn)
3816 {
3817 switch (fn)
3818 {
3819 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3820 case IFN_##CODE: break;
3821 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3822 case IFN_##CODE: return OPTAB##_optab;
3823 #include "internal-fn.def"
3824
3825 case IFN_LAST:
3826 break;
3827 }
3828 gcc_unreachable ();
3829 }
3830
3831 /* Return true if FN is supported for the types in TYPES when the
3832 optimization type is OPT_TYPE. The types are those associated with
3833 the "type0" and "type1" fields of FN's direct_internal_fn_info
3834 structure. */
3835
3836 bool
3837 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
3838 optimization_type opt_type)
3839 {
3840 switch (fn)
3841 {
3842 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3843 case IFN_##CODE: break;
3844 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3845 case IFN_##CODE: \
3846 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3847 opt_type);
3848 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3849 UNSIGNED_OPTAB, TYPE) \
3850 case IFN_##CODE: \
3851 { \
3852 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3853 ? UNSIGNED_OPTAB ## _optab \
3854 : SIGNED_OPTAB ## _optab); \
3855 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3856 opt_type); \
3857 }
3858 #include "internal-fn.def"
3859
3860 case IFN_LAST:
3861 break;
3862 }
3863 gcc_unreachable ();
3864 }
3865
3866 /* Return true if FN is supported for type TYPE when the optimization
3867 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3868 fields of FN's direct_internal_fn_info structure are the same. */
3869
3870 bool
3871 direct_internal_fn_supported_p (internal_fn fn, tree type,
3872 optimization_type opt_type)
3873 {
3874 const direct_internal_fn_info &info = direct_internal_fn (fn);
3875 gcc_checking_assert (info.type0 == info.type1);
3876 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
3877 }
3878
3879 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3880 given that STMT is a call to a direct internal function. */
3881
3882 bool
3883 direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
3884 {
3885 internal_fn fn = gimple_call_internal_fn (stmt);
3886 tree_pair types = direct_internal_fn_types (fn, stmt);
3887 return direct_internal_fn_supported_p (fn, types, opt_type);
3888 }
3889
3890 /* Return true if FN is a binary operation and if FN is commutative. */
3891
3892 bool
3893 commutative_binary_fn_p (internal_fn fn)
3894 {
3895 switch (fn)
3896 {
3897 case IFN_AVG_FLOOR:
3898 case IFN_AVG_CEIL:
3899 case IFN_MULH:
3900 case IFN_MULHS:
3901 case IFN_MULHRS:
3902 case IFN_FMIN:
3903 case IFN_FMAX:
3904 case IFN_COMPLEX_MUL:
3905 case IFN_UBSAN_CHECK_ADD:
3906 case IFN_UBSAN_CHECK_MUL:
3907 case IFN_ADD_OVERFLOW:
3908 case IFN_MUL_OVERFLOW:
3909 return true;
3910
3911 default:
3912 return false;
3913 }
3914 }
3915
3916 /* Return true if FN is a ternary operation and if its first two arguments
3917 are commutative. */
3918
3919 bool
3920 commutative_ternary_fn_p (internal_fn fn)
3921 {
3922 switch (fn)
3923 {
3924 case IFN_FMA:
3925 case IFN_FMS:
3926 case IFN_FNMA:
3927 case IFN_FNMS:
3928 return true;
3929
3930 default:
3931 return false;
3932 }
3933 }
3934
3935 /* Return true if FN is an associative binary operation. */
3936
3937 bool
3938 associative_binary_fn_p (internal_fn fn)
3939 {
3940 switch (fn)
3941 {
3942 case IFN_FMIN:
3943 case IFN_FMAX:
3944 return true;
3945
3946 default:
3947 return false;
3948 }
3949 }
3950
3951 /* If FN is commutative in two consecutive arguments, return the
3952 index of the first, otherwise return -1. */
3953
3954 int
3955 first_commutative_argument (internal_fn fn)
3956 {
3957 switch (fn)
3958 {
3959 case IFN_COND_ADD:
3960 case IFN_COND_MUL:
3961 case IFN_COND_MIN:
3962 case IFN_COND_MAX:
3963 case IFN_COND_FMIN:
3964 case IFN_COND_FMAX:
3965 case IFN_COND_AND:
3966 case IFN_COND_IOR:
3967 case IFN_COND_XOR:
3968 case IFN_COND_FMA:
3969 case IFN_COND_FMS:
3970 case IFN_COND_FNMA:
3971 case IFN_COND_FNMS:
3972 return 1;
3973
3974 default:
3975 if (commutative_binary_fn_p (fn)
3976 || commutative_ternary_fn_p (fn))
3977 return 0;
3978 return -1;
3979 }
3980 }
3981
3982 /* Return true if IFN_SET_EDOM is supported. */
3983
3984 bool
3985 set_edom_supported_p (void)
3986 {
3987 #ifdef TARGET_EDOM
3988 return true;
3989 #else
3990 return false;
3991 #endif
3992 }
3993
3994 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3995 static void \
3996 expand_##CODE (internal_fn fn, gcall *stmt) \
3997 { \
3998 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3999 }
4000 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4001 UNSIGNED_OPTAB, TYPE) \
4002 static void \
4003 expand_##CODE (internal_fn fn, gcall *stmt) \
4004 { \
4005 tree_pair types = direct_internal_fn_types (fn, stmt); \
4006 optab which_optab = direct_internal_fn_optab (fn, types); \
4007 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4008 }
4009 #include "internal-fn.def"
4010
4011 /* Routines to expand each internal function, indexed by function number.
4012 Each routine has the prototype:
4013
4014 expand_<NAME> (gcall *stmt)
4015
4016 where STMT is the statement that performs the call. */
4017 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
4018 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4019 #include "internal-fn.def"
4020 0
4021 };
4022
4023 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
4024 tree code CODE. */
4025 #define FOR_EACH_CODE_MAPPING(T) \
4026 T (PLUS_EXPR, IFN_COND_ADD) \
4027 T (MINUS_EXPR, IFN_COND_SUB) \
4028 T (MULT_EXPR, IFN_COND_MUL) \
4029 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
4030 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
4031 T (RDIV_EXPR, IFN_COND_RDIV) \
4032 T (MIN_EXPR, IFN_COND_MIN) \
4033 T (MAX_EXPR, IFN_COND_MAX) \
4034 T (BIT_AND_EXPR, IFN_COND_AND) \
4035 T (BIT_IOR_EXPR, IFN_COND_IOR) \
4036 T (BIT_XOR_EXPR, IFN_COND_XOR) \
4037 T (LSHIFT_EXPR, IFN_COND_SHL) \
4038 T (RSHIFT_EXPR, IFN_COND_SHR) \
4039 T (NEGATE_EXPR, IFN_COND_NEG)
4040
4041 /* Return a function that only performs CODE when a certain condition is met
4042 and that uses a given fallback value otherwise. For example, if CODE is
4043 a binary operation associated with conditional function FN:
4044
4045 LHS = FN (COND, A, B, ELSE)
4046
4047 is equivalent to the C expression:
4048
4049 LHS = COND ? A CODE B : ELSE;
4050
4051 operating elementwise if the operands are vectors.
4052
4053 Return IFN_LAST if no such function exists. */
4054
4055 internal_fn
4056 get_conditional_internal_fn (tree_code code)
4057 {
4058 switch (code)
4059 {
4060 #define CASE(CODE, IFN) case CODE: return IFN;
4061 FOR_EACH_CODE_MAPPING(CASE)
4062 #undef CASE
4063 default:
4064 return IFN_LAST;
4065 }
4066 }
4067
4068 /* If IFN implements the conditional form of a tree code, return that
4069 tree code, otherwise return ERROR_MARK. */
4070
4071 tree_code
4072 conditional_internal_fn_code (internal_fn ifn)
4073 {
4074 switch (ifn)
4075 {
4076 #define CASE(CODE, IFN) case IFN: return CODE;
4077 FOR_EACH_CODE_MAPPING(CASE)
4078 #undef CASE
4079 default:
4080 return ERROR_MARK;
4081 }
4082 }
4083
4084 /* Invoke T(IFN) for each internal function IFN that also has an
4085 IFN_COND_* form. */
4086 #define FOR_EACH_COND_FN_PAIR(T) \
4087 T (FMAX) \
4088 T (FMIN) \
4089 T (FMA) \
4090 T (FMS) \
4091 T (FNMA) \
4092 T (FNMS)
4093
4094 /* Return a function that only performs internal function FN when a
4095 certain condition is met and that uses a given fallback value otherwise.
4096 In other words, the returned function FN' is such that:
4097
4098 LHS = FN' (COND, A1, ... An, ELSE)
4099
4100 is equivalent to the C expression:
4101
4102 LHS = COND ? FN (A1, ..., An) : ELSE;
4103
4104 operating elementwise if the operands are vectors.
4105
4106 Return IFN_LAST if no such function exists. */
4107
4108 internal_fn
4109 get_conditional_internal_fn (internal_fn fn)
4110 {
4111 switch (fn)
4112 {
4113 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4114 FOR_EACH_COND_FN_PAIR(CASE)
4115 #undef CASE
4116 default:
4117 return IFN_LAST;
4118 }
4119 }
4120
4121 /* If IFN implements the conditional form of an unconditional internal
4122 function, return that unconditional function, otherwise return IFN_LAST. */
4123
4124 internal_fn
4125 get_unconditional_internal_fn (internal_fn ifn)
4126 {
4127 switch (ifn)
4128 {
4129 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
4130 FOR_EACH_COND_FN_PAIR(CASE)
4131 #undef CASE
4132 default:
4133 return IFN_LAST;
4134 }
4135 }
4136
4137 /* Return true if STMT can be interpreted as a conditional tree code
4138 operation of the form:
4139
4140 LHS = COND ? OP (RHS1, ...) : ELSE;
4141
4142 operating elementwise if the operands are vectors. This includes
4143 the case of an all-true COND, so that the operation always happens.
4144
4145 When returning true, set:
4146
4147 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4148 is known to be all-true
4149 - *CODE_OUT to the tree code
4150 - OPS[I] to operand I of *CODE_OUT
4151 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4152 condition is known to be all true. */
4153
4154 bool
4155 can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
4156 tree_code *code_out,
4157 tree (&ops)[3], tree *else_out)
4158 {
4159 if (gassign *assign = dyn_cast <gassign *> (stmt))
4160 {
4161 *cond_out = NULL_TREE;
4162 *code_out = gimple_assign_rhs_code (assign);
4163 ops[0] = gimple_assign_rhs1 (assign);
4164 ops[1] = gimple_assign_rhs2 (assign);
4165 ops[2] = gimple_assign_rhs3 (assign);
4166 *else_out = NULL_TREE;
4167 return true;
4168 }
4169 if (gcall *call = dyn_cast <gcall *> (stmt))
4170 if (gimple_call_internal_p (call))
4171 {
4172 internal_fn ifn = gimple_call_internal_fn (call);
4173 tree_code code = conditional_internal_fn_code (ifn);
4174 if (code != ERROR_MARK)
4175 {
4176 *cond_out = gimple_call_arg (call, 0);
4177 *code_out = code;
4178 unsigned int nops = gimple_call_num_args (call) - 2;
4179 for (unsigned int i = 0; i < 3; ++i)
4180 ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
4181 *else_out = gimple_call_arg (call, nops + 1);
4182 if (integer_truep (*cond_out))
4183 {
4184 *cond_out = NULL_TREE;
4185 *else_out = NULL_TREE;
4186 }
4187 return true;
4188 }
4189 }
4190 return false;
4191 }
4192
4193 /* Return true if IFN is some form of load from memory. */
4194
4195 bool
4196 internal_load_fn_p (internal_fn fn)
4197 {
4198 switch (fn)
4199 {
4200 case IFN_MASK_LOAD:
4201 case IFN_LOAD_LANES:
4202 case IFN_MASK_LOAD_LANES:
4203 case IFN_GATHER_LOAD:
4204 case IFN_MASK_GATHER_LOAD:
4205 case IFN_LEN_LOAD:
4206 return true;
4207
4208 default:
4209 return false;
4210 }
4211 }
4212
4213 /* Return true if IFN is some form of store to memory. */
4214
4215 bool
4216 internal_store_fn_p (internal_fn fn)
4217 {
4218 switch (fn)
4219 {
4220 case IFN_MASK_STORE:
4221 case IFN_STORE_LANES:
4222 case IFN_MASK_STORE_LANES:
4223 case IFN_SCATTER_STORE:
4224 case IFN_MASK_SCATTER_STORE:
4225 case IFN_LEN_STORE:
4226 return true;
4227
4228 default:
4229 return false;
4230 }
4231 }
4232
4233 /* Return true if IFN is some form of gather load or scatter store. */
4234
4235 bool
4236 internal_gather_scatter_fn_p (internal_fn fn)
4237 {
4238 switch (fn)
4239 {
4240 case IFN_GATHER_LOAD:
4241 case IFN_MASK_GATHER_LOAD:
4242 case IFN_SCATTER_STORE:
4243 case IFN_MASK_SCATTER_STORE:
4244 return true;
4245
4246 default:
4247 return false;
4248 }
4249 }
4250
4251 /* If FN takes a vector mask argument, return the index of that argument,
4252 otherwise return -1. */
4253
4254 int
4255 internal_fn_mask_index (internal_fn fn)
4256 {
4257 switch (fn)
4258 {
4259 case IFN_MASK_LOAD:
4260 case IFN_MASK_LOAD_LANES:
4261 case IFN_MASK_STORE:
4262 case IFN_MASK_STORE_LANES:
4263 return 2;
4264
4265 case IFN_MASK_GATHER_LOAD:
4266 case IFN_MASK_SCATTER_STORE:
4267 return 4;
4268
4269 default:
4270 return (conditional_internal_fn_code (fn) != ERROR_MARK
4271 || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
4272 }
4273 }
4274
4275 /* If FN takes a value that should be stored to memory, return the index
4276 of that argument, otherwise return -1. */
4277
4278 int
4279 internal_fn_stored_value_index (internal_fn fn)
4280 {
4281 switch (fn)
4282 {
4283 case IFN_MASK_STORE:
4284 case IFN_MASK_STORE_LANES:
4285 case IFN_SCATTER_STORE:
4286 case IFN_MASK_SCATTER_STORE:
4287 case IFN_LEN_STORE:
4288 return 3;
4289
4290 default:
4291 return -1;
4292 }
4293 }
4294
4295 /* Return true if the target supports gather load or scatter store function
4296 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4297 while for stores it is the vector type of the stored data argument.
4298 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4299 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4300 offset from the shared base address of each loaded or stored element.
4301 SCALE is the amount by which these offsets should be multiplied
4302 *after* they have been extended to address width. */
4303
4304 bool
4305 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
4306 tree memory_element_type,
4307 tree offset_vector_type, int scale)
4308 {
4309 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
4310 TYPE_SIZE (memory_element_type)))
4311 return false;
4312 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type),
4313 TYPE_VECTOR_SUBPARTS (offset_vector_type)))
4314 return false;
4315 optab optab = direct_internal_fn_optab (ifn);
4316 insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type),
4317 TYPE_MODE (offset_vector_type));
4318 int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
4319 bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
4320 return (icode != CODE_FOR_nothing
4321 && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
4322 && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale)));
4323 }
4324
4325 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4326 for pointers of type TYPE when the accesses have LENGTH bytes and their
4327 common byte alignment is ALIGN. */
4328
4329 bool
4330 internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
4331 poly_uint64 length, unsigned int align)
4332 {
4333 machine_mode mode = TYPE_MODE (type);
4334 optab optab = direct_internal_fn_optab (ifn);
4335 insn_code icode = direct_optab_handler (optab, mode);
4336 if (icode == CODE_FOR_nothing)
4337 return false;
4338 rtx length_rtx = immed_wide_int_const (length, mode);
4339 return (insn_operand_matches (icode, 3, length_rtx)
4340 && insn_operand_matches (icode, 4, GEN_INT (align)));
4341 }
4342
4343 /* Return the supported bias for IFN which is either IFN_LEN_LOAD
4344 or IFN_LEN_STORE. For now we only support the biases of 0 and -1
4345 (in case 0 is not an allowable length for len_load or len_store).
4346 If none of the biases match what the backend provides, return
4347 VECT_PARTIAL_BIAS_UNSUPPORTED. */
4348
4349 signed char
4350 internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
4351 {
4352 optab optab = direct_internal_fn_optab (ifn);
4353 insn_code icode = direct_optab_handler (optab, mode);
4354
4355 if (icode != CODE_FOR_nothing)
4356 {
4357 /* For now we only support biases of 0 or -1. Try both of them. */
4358 if (insn_operand_matches (icode, 3, GEN_INT (0)))
4359 return 0;
4360 if (insn_operand_matches (icode, 3, GEN_INT (-1)))
4361 return -1;
4362 }
4363
4364 return VECT_PARTIAL_BIAS_UNSUPPORTED;
4365 }
4366
4367 /* Expand STMT as though it were a call to internal function FN. */
4368
4369 void
4370 expand_internal_call (internal_fn fn, gcall *stmt)
4371 {
4372 internal_fn_expanders[fn] (fn, stmt);
4373 }
4374
4375 /* Expand STMT, which is a call to internal function FN. */
4376
4377 void
4378 expand_internal_call (gcall *stmt)
4379 {
4380 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
4381 }
4382
4383 /* If TYPE is a vector type, return true if IFN is a direct internal
4384 function that is supported for that type. If TYPE is a scalar type,
4385 return true if IFN is a direct internal function that is supported for
4386 the target's preferred vector version of TYPE. */
4387
4388 bool
4389 vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
4390 {
4391 if (VECTOR_MODE_P (TYPE_MODE (type)))
4392 return direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED);
4393
4394 scalar_mode smode;
4395 if (VECTOR_TYPE_P (type)
4396 || !is_a <scalar_mode> (TYPE_MODE (type), &smode))
4397 return false;
4398
4399 machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
4400 if (VECTOR_MODE_P (vmode))
4401 {
4402 tree vectype = build_vector_type_for_mode (type, vmode);
4403 if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4404 return true;
4405 }
4406
4407 auto_vector_modes vector_modes;
4408 targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
4409 for (machine_mode base_mode : vector_modes)
4410 if (related_vector_mode (base_mode, smode).exists (&vmode))
4411 {
4412 tree vectype = build_vector_type_for_mode (type, vmode);
4413 if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4414 return true;
4415 }
4416
4417 return false;
4418 }
4419
4420 void
4421 expand_SHUFFLEVECTOR (internal_fn, gcall *)
4422 {
4423 gcc_unreachable ();
4424 }
4425
4426 void
4427 expand_PHI (internal_fn, gcall *)
4428 {
4429 gcc_unreachable ();
4430 }
4431
4432 void
4433 expand_SPACESHIP (internal_fn, gcall *stmt)
4434 {
4435 tree lhs = gimple_call_lhs (stmt);
4436 tree rhs1 = gimple_call_arg (stmt, 0);
4437 tree rhs2 = gimple_call_arg (stmt, 1);
4438 tree type = TREE_TYPE (rhs1);
4439
4440 do_pending_stack_adjust ();
4441
4442 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
4443 rtx op1 = expand_normal (rhs1);
4444 rtx op2 = expand_normal (rhs2);
4445
4446 class expand_operand ops[3];
4447 create_output_operand (&ops[0], target, TYPE_MODE (TREE_TYPE (lhs)));
4448 create_input_operand (&ops[1], op1, TYPE_MODE (type));
4449 create_input_operand (&ops[2], op2, TYPE_MODE (type));
4450 insn_code icode = optab_handler (spaceship_optab, TYPE_MODE (type));
4451 expand_insn (icode, 3, ops);
4452 if (!rtx_equal_p (target, ops[0].value))
4453 emit_move_insn (target, ops[0].value);
4454 }
This page took 0.249975 seconds and 5 git commands to generate.