]>
Commit | Line | Data |
---|---|---|
8c660648 | 1 | /* Instruction scheduling pass. |
a5544970 | 2 | Copyright (C) 1992-2019 Free Software Foundation, Inc. |
8c660648 JL |
3 | Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by, |
4 | and currently maintained by, Jim Wilson (wilson@cygnus.com) | |
5 | ||
1322177d | 6 | This file is part of GCC. |
5d14e356 | 7 | |
1322177d LB |
8 | GCC is free software; you can redistribute it and/or modify it under |
9 | the terms of the GNU General Public License as published by the Free | |
9dcd6f09 | 10 | Software Foundation; either version 3, or (at your option) any later |
1322177d | 11 | version. |
5d14e356 | 12 | |
1322177d LB |
13 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
5d14e356 RK |
15 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
16 | for more details. | |
17 | ||
18 | You should have received a copy of the GNU General Public License | |
9dcd6f09 NC |
19 | along with GCC; see the file COPYING3. If not see |
20 | <http://www.gnu.org/licenses/>. */ | |
8c660648 | 21 | |
b4ead7d4 | 22 | /* Instruction scheduling pass. This file, along with sched-deps.c, |
151e9aac | 23 | contains the generic parts. The actual entry point for |
b4ead7d4 | 24 | the normal instruction scheduling pass is found in sched-rgn.c. |
8c660648 JL |
25 | |
26 | We compute insn priorities based on data dependencies. Flow | |
27 | analysis only creates a fraction of the data-dependencies we must | |
28 | observe: namely, only those dependencies which the combiner can be | |
29 | expected to use. For this pass, we must therefore create the | |
30 | remaining dependencies we need to observe: register dependencies, | |
31 | memory dependencies, dependencies to keep function calls in order, | |
32 | and the dependence between a conditional branch and the setting of | |
33 | condition codes are all dealt with here. | |
34 | ||
35 | The scheduler first traverses the data flow graph, starting with | |
36 | the last instruction, and proceeding to the first, assigning values | |
37 | to insn_priority as it goes. This sorts the instructions | |
38 | topologically by data dependence. | |
39 | ||
40 | Once priorities have been established, we order the insns using | |
41 | list scheduling. This works as follows: starting with a list of | |
42 | all the ready insns, and sorted according to priority number, we | |
43 | schedule the insn from the end of the list by placing its | |
44 | predecessors in the list according to their priority order. We | |
45 | consider this insn scheduled by setting the pointer to the "end" of | |
46 | the list to point to the previous insn. When an insn has no | |
47 | predecessors, we either queue it until sufficient time has elapsed | |
48 | or add it to the ready list. As the instructions are scheduled or | |
49 | when stalls are introduced, the queue advances and dumps insns into | |
50 | the ready list. When all insns down to the lowest priority have | |
51 | been scheduled, the critical path of the basic block has been made | |
52 | as short as possible. The remaining insns are then scheduled in | |
53 | remaining slots. | |
54 | ||
8c660648 JL |
55 | The following list shows the order in which we want to break ties |
56 | among insns in the ready list: | |
57 | ||
58 | 1. choose insn with the longest path to end of bb, ties | |
59 | broken by | |
60 | 2. choose insn with least contribution to register pressure, | |
61 | ties broken by | |
62 | 3. prefer in-block upon interblock motion, ties broken by | |
63 | 4. prefer useful upon speculative motion, ties broken by | |
64 | 5. choose insn with largest control flow probability, ties | |
65 | broken by | |
66 | 6. choose insn with the least dependences upon the previously | |
67 | scheduled insn, or finally | |
2db45993 JL |
68 | 7 choose the insn which has the most insns dependent on it. |
69 | 8. choose insn with lowest UID. | |
8c660648 JL |
70 | |
71 | Memory references complicate matters. Only if we can be certain | |
72 | that memory references are not part of the data dependency graph | |
73 | (via true, anti, or output dependence), can we move operations past | |
74 | memory references. To first approximation, reads can be done | |
75 | independently, while writes introduce dependencies. Better | |
76 | approximations will yield fewer dependencies. | |
77 | ||
78 | Before reload, an extended analysis of interblock data dependences | |
79 | is required for interblock scheduling. This is performed in | |
151e9aac | 80 | compute_block_dependences (). |
8c660648 JL |
81 | |
82 | Dependencies set up by memory references are treated in exactly the | |
b198261f MK |
83 | same way as other dependencies, by using insn backward dependences |
84 | INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences | |
151e9aac | 85 | INSN_FORW_DEPS for the purpose of forward list scheduling. |
8c660648 JL |
86 | |
87 | Having optimized the critical path, we may have also unduly | |
88 | extended the lifetimes of some registers. If an operation requires | |
89 | that constants be loaded into registers, it is certainly desirable | |
90 | to load those constants as early as necessary, but no earlier. | |
91 | I.e., it will not do to load up a bunch of registers at the | |
92 | beginning of a basic block only to use them at the end, if they | |
93 | could be loaded later, since this may result in excessive register | |
94 | utilization. | |
95 | ||
96 | Note that since branches are never in basic blocks, but only end | |
97 | basic blocks, this pass will not move branches. But that is ok, | |
98 | since we can use GNU's delayed branch scheduling pass to take care | |
99 | of this case. | |
100 | ||
101 | Also note that no further optimizations based on algebraic | |
102 | identities are performed, so this pass would be a good one to | |
103 | perform instruction splitting, such as breaking up a multiply | |
104 | instruction into shifts and adds where that is profitable. | |
105 | ||
106 | Given the memory aliasing analysis that this pass should perform, | |
107 | it should be possible to remove redundant stores to memory, and to | |
108 | load values from registers instead of hitting memory. | |
109 | ||
110 | Before reload, speculative insns are moved only if a 'proof' exists | |
111 | that no exception will be caused by this, and if no live registers | |
112 | exist that inhibit the motion (live registers constraints are not | |
113 | represented by data dependence edges). | |
114 | ||
115 | This pass must update information that subsequent passes expect to | |
116 | be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths, | |
a813c111 | 117 | reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END. |
8c660648 JL |
118 | |
119 | The information in the line number notes is carefully retained by | |
120 | this pass. Notes that refer to the starting and ending of | |
121 | exception regions are also carefully retained by this pass. All | |
122 | other NOTE insns are grouped in their same relative order at the | |
b4ead7d4 | 123 | beginning of basic blocks and regions that have been scheduled. */ |
8c660648 | 124 | \f |
8c660648 | 125 | #include "config.h" |
5835e573 | 126 | #include "system.h" |
4977bab6 | 127 | #include "coretypes.h" |
c7131fb2 | 128 | #include "backend.h" |
957060b5 | 129 | #include "target.h" |
8c660648 | 130 | #include "rtl.h" |
957060b5 | 131 | #include "cfghooks.h" |
c7131fb2 | 132 | #include "df.h" |
4d0cdd0c | 133 | #include "memmodel.h" |
6baf1cc8 | 134 | #include "tm_p.h" |
957060b5 | 135 | #include "insn-config.h" |
8c660648 | 136 | #include "regs.h" |
957060b5 AM |
137 | #include "ira.h" |
138 | #include "recog.h" | |
8c660648 | 139 | #include "insn-attr.h" |
60393bbc AM |
140 | #include "cfgrtl.h" |
141 | #include "cfgbuild.h" | |
1708fd40 | 142 | #include "sched-int.h" |
677f3fa8 | 143 | #include "common/common-target.h" |
496d7bb0 | 144 | #include "params.h" |
6fb5fa3c | 145 | #include "dbgcnt.h" |
e855c69d | 146 | #include "cfgloop.h" |
7ee2468b | 147 | #include "dumpfile.h" |
013a8899 | 148 | #include "print-rtl.h" |
8c660648 | 149 | |
8c660648 JL |
150 | #ifdef INSN_SCHEDULING |
151 | ||
f20f2613 VM |
152 | /* True if we do register pressure relief through live-range |
153 | shrinkage. */ | |
154 | static bool live_range_shrinkage_p; | |
155 | ||
156 | /* Switch on live range shrinkage. */ | |
157 | void | |
158 | initialize_live_range_shrinkage (void) | |
159 | { | |
160 | live_range_shrinkage_p = true; | |
161 | } | |
162 | ||
163 | /* Switch off live range shrinkage. */ | |
164 | void | |
165 | finish_live_range_shrinkage (void) | |
166 | { | |
167 | live_range_shrinkage_p = false; | |
168 | } | |
169 | ||
8c660648 JL |
170 | /* issue_rate is the number of insns that can be scheduled in the same |
171 | machine cycle. It can be defined in the config/mach/mach.h file, | |
172 | otherwise we set it to 1. */ | |
173 | ||
e855c69d | 174 | int issue_rate; |
8c660648 | 175 | |
26965010 BS |
176 | /* This can be set to true by a backend if the scheduler should not |
177 | enable a DCE pass. */ | |
178 | bool sched_no_dce; | |
179 | ||
06d7e8e7 BS |
180 | /* The current initiation interval used when modulo scheduling. */ |
181 | static int modulo_ii; | |
182 | ||
183 | /* The maximum number of stages we are prepared to handle. */ | |
184 | static int modulo_max_stages; | |
185 | ||
186 | /* The number of insns that exist in each iteration of the loop. We use this | |
187 | to detect when we've scheduled all insns from the first iteration. */ | |
188 | static int modulo_n_insns; | |
189 | ||
190 | /* The current count of insns in the first iteration of the loop that have | |
191 | already been scheduled. */ | |
192 | static int modulo_insns_scheduled; | |
193 | ||
194 | /* The maximum uid of insns from the first iteration of the loop. */ | |
195 | static int modulo_iter0_max_uid; | |
196 | ||
197 | /* The number of times we should attempt to backtrack when modulo scheduling. | |
198 | Decreased each time we have to backtrack. */ | |
199 | static int modulo_backtracks_left; | |
200 | ||
201 | /* The stage in which the last insn from the original loop was | |
202 | scheduled. */ | |
203 | static int modulo_last_stage; | |
204 | ||
cc132865 | 205 | /* sched-verbose controls the amount of debugging output the |
409f8483 | 206 | scheduler prints. It is controlled by -fsched-verbose=N: |
4111f1c9 NB |
207 | N=0: no debugging output. |
208 | N=1: default value. | |
8c660648 JL |
209 | N=2: bb's probabilities, detailed ready list info, unit/insn info. |
210 | N=3: rtl at abort point, control-flow, regions info. | |
cc132865 | 211 | N=5: dependences info. */ |
b4ead7d4 | 212 | int sched_verbose = 0; |
8c660648 | 213 | |
4111f1c9 | 214 | /* Debugging file. All printouts are sent to dump. */ |
c62c2659 | 215 | FILE *sched_dump = 0; |
a88f02e7 | 216 | |
b8698a0f | 217 | /* This is a placeholder for the scheduler parameters common |
e855c69d AB |
218 | to all schedulers. */ |
219 | struct common_sched_info_def *common_sched_info; | |
f66d83e1 | 220 | |
e855c69d | 221 | #define INSN_TICK(INSN) (HID (INSN)->tick) |
26965010 BS |
222 | #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick) |
223 | #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate) | |
e855c69d | 224 | #define INTER_TICK(INSN) (HID (INSN)->inter_tick) |
26965010 BS |
225 | #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn) |
226 | #define SHADOW_P(INSN) (HID (INSN)->shadow_p) | |
e2724e63 | 227 | #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec) |
ffc1ded5 | 228 | /* Cached cost of the instruction. Use insn_sched_cost to get cost of the |
1a83e602 BS |
229 | insn. -1 here means that the field is not initialized. */ |
230 | #define INSN_COST(INSN) (HID (INSN)->cost) | |
63f54b1a MK |
231 | |
232 | /* If INSN_TICK of an instruction is equal to INVALID_TICK, | |
233 | then it should be recalculated from scratch. */ | |
234 | #define INVALID_TICK (-(max_insn_queue_index + 1)) | |
235 | /* The minimal value of the INSN_TICK of an instruction. */ | |
236 | #define MIN_TICK (-max_insn_queue_index) | |
8c660648 | 237 | |
63387a85 MK |
238 | /* Original order of insns in the ready list. |
239 | Used to keep order of normal insns while separating DEBUG_INSNs. */ | |
240 | #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order) | |
241 | ||
b75ae7f4 MK |
242 | /* The deciding reason for INSN's place in the ready list. */ |
243 | #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win) | |
244 | ||
8c660648 JL |
245 | /* List of important notes we must keep around. This is a pointer to the |
246 | last element in the list. */ | |
ce1ce33a | 247 | rtx_insn *note_list; |
8c660648 | 248 | |
496d7bb0 MK |
249 | static struct spec_info_def spec_info_var; |
250 | /* Description of the speculative part of the scheduling. | |
251 | If NULL - no speculation. */ | |
e855c69d | 252 | spec_info_t spec_info = NULL; |
496d7bb0 MK |
253 | |
254 | /* True, if recovery block was added during scheduling of current block. | |
255 | Used to determine, if we need to fix INSN_TICKs. */ | |
e2f6ff94 MK |
256 | static bool haifa_recovery_bb_recently_added_p; |
257 | ||
258 | /* True, if recovery block was added during this scheduling pass. | |
259 | Used to determine if we should have empty memory pools of dependencies | |
260 | after finishing current region. */ | |
261 | bool haifa_recovery_bb_ever_added_p; | |
496d7bb0 | 262 | |
917f1b7e | 263 | /* Counters of different types of speculative instructions. */ |
496d7bb0 MK |
264 | static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control; |
265 | ||
496d7bb0 | 266 | /* Array used in {unlink, restore}_bb_notes. */ |
9b517712 | 267 | static rtx_insn **bb_header = 0; |
496d7bb0 | 268 | |
496d7bb0 MK |
269 | /* Basic block after which recovery blocks will be created. */ |
270 | static basic_block before_recovery; | |
271 | ||
e855c69d AB |
272 | /* Basic block just before the EXIT_BLOCK and after recovery, if we have |
273 | created it. */ | |
274 | basic_block after_recovery; | |
275 | ||
276 | /* FALSE if we add bb to another region, so we don't need to initialize it. */ | |
277 | bool adding_bb_to_current_region_p = true; | |
278 | ||
8c660648 JL |
279 | /* Queues, etc. */ |
280 | ||
281 | /* An instruction is ready to be scheduled when all insns preceding it | |
282 | have already been scheduled. It is important to ensure that all | |
283 | insns which use its result will not be executed until its result | |
284 | has been computed. An insn is maintained in one of four structures: | |
285 | ||
286 | (P) the "Pending" set of insns which cannot be scheduled until | |
287 | their dependencies have been satisfied. | |
288 | (Q) the "Queued" set of insns that can be scheduled when sufficient | |
289 | time has passed. | |
290 | (R) the "Ready" list of unscheduled, uncommitted insns. | |
291 | (S) the "Scheduled" list of insns. | |
292 | ||
293 | Initially, all insns are either "Pending" or "Ready" depending on | |
294 | whether their dependencies are satisfied. | |
295 | ||
296 | Insns move from the "Ready" list to the "Scheduled" list as they | |
297 | are committed to the schedule. As this occurs, the insns in the | |
298 | "Pending" list have their dependencies satisfied and move to either | |
299 | the "Ready" list or the "Queued" set depending on whether | |
300 | sufficient time has passed to make them ready. As time passes, | |
fa0aee89 | 301 | insns move from the "Queued" set to the "Ready" list. |
8c660648 | 302 | |
b198261f MK |
303 | The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the |
304 | unscheduled insns, i.e., those that are ready, queued, and pending. | |
8c660648 JL |
305 | The "Queued" set (Q) is implemented by the variable `insn_queue'. |
306 | The "Ready" list (R) is implemented by the variables `ready' and | |
307 | `n_ready'. | |
308 | The "Scheduled" list (S) is the new insn chain built by this pass. | |
309 | ||
310 | The transition (R->S) is implemented in the scheduling loop in | |
311 | `schedule_block' when the best insn to schedule is chosen. | |
8c660648 JL |
312 | The transitions (P->R and P->Q) are implemented in `schedule_insn' as |
313 | insns move from the ready list to the scheduled list. | |
314 | The transition (Q->R) is implemented in 'queue_to_insn' as time | |
315 | passes or stalls are introduced. */ | |
316 | ||
317 | /* Implement a circular buffer to delay instructions until sufficient | |
fa0aee89 | 318 | time has passed. For the new pipeline description interface, |
63f54b1a | 319 | MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less |
fa0aee89 PB |
320 | than maximal time of instruction execution computed by genattr.c on |
321 | the base maximal time of functional unit reservations and getting a | |
322 | result. This is the longest time an insn may be queued. */ | |
fae15c93 | 323 | |
3dc99c19 | 324 | static rtx_insn_list **insn_queue; |
8c660648 JL |
325 | static int q_ptr = 0; |
326 | static int q_size = 0; | |
fa0aee89 PB |
327 | #define NEXT_Q(X) (((X)+1) & max_insn_queue_index) |
328 | #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index) | |
fae15c93 | 329 | |
63f54b1a MK |
330 | #define QUEUE_SCHEDULED (-3) |
331 | #define QUEUE_NOWHERE (-2) | |
332 | #define QUEUE_READY (-1) | |
333 | /* QUEUE_SCHEDULED - INSN is scheduled. | |
334 | QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in | |
335 | queue or ready list. | |
336 | QUEUE_READY - INSN is in ready list. | |
337 | N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */ | |
b8698a0f | 338 | |
e855c69d | 339 | #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index) |
63f54b1a | 340 | |
fae15c93 VM |
341 | /* The following variable value refers for all current and future |
342 | reservations of the processor units. */ | |
343 | state_t curr_state; | |
344 | ||
345 | /* The following variable value is size of memory representing all | |
fa0aee89 | 346 | current and future reservations of the processor units. */ |
e855c69d | 347 | size_t dfa_state_size; |
fae15c93 VM |
348 | |
349 | /* The following array is used to find the best insn from ready when | |
350 | the automaton pipeline interface is used. */ | |
4960a0cb | 351 | signed char *ready_try = NULL; |
8c660648 | 352 | |
e855c69d | 353 | /* The ready list. */ |
b5b8b0ac | 354 | struct ready_list ready = {NULL, 0, 0, 0, 0}; |
176f9a7b | 355 | |
e855c69d AB |
356 | /* The pointer to the ready list (to be removed). */ |
357 | static struct ready_list *readyp = &ready; | |
63f54b1a MK |
358 | |
359 | /* Scheduling clock. */ | |
360 | static int clock_var; | |
361 | ||
26965010 BS |
362 | /* Clock at which the previous instruction was issued. */ |
363 | static int last_clock_var; | |
364 | ||
365 | /* Set to true if, when queuing a shadow insn, we discover that it would be | |
366 | scheduled too late. */ | |
367 | static bool must_backtrack; | |
368 | ||
369 | /* The following variable value is number of essential insns issued on | |
370 | the current cycle. An insn is essential one if it changes the | |
371 | processors state. */ | |
372 | int cycle_issued_insns; | |
373 | ||
86014d07 BS |
374 | /* This records the actual schedule. It is built up during the main phase |
375 | of schedule_block, and afterwards used to reorder the insns in the RTL. */ | |
ce1ce33a | 376 | static vec<rtx_insn *> scheduled_insns; |
86014d07 | 377 | |
9678086d | 378 | static int may_trap_exp (const_rtx, int); |
15aab9c0 VM |
379 | |
380 | /* Nonzero iff the address is comprised from at most 1 register. */ | |
381 | #define CONST_BASED_ADDRESS_P(x) \ | |
f8cfc6aa | 382 | (REG_P (x) \ |
15aab9c0 VM |
383 | || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \ |
384 | || (GET_CODE (x) == LO_SUM)) \ | |
385 | && (CONSTANT_P (XEXP (x, 0)) \ | |
386 | || CONSTANT_P (XEXP (x, 1))))) | |
387 | ||
388 | /* Returns a class that insn with GET_DEST(insn)=x may belong to, | |
389 | as found by analyzing insn's expression. */ | |
390 | ||
e855c69d AB |
391 | \f |
392 | static int haifa_luid_for_non_insn (rtx x); | |
393 | ||
394 | /* Haifa version of sched_info hooks common to all headers. */ | |
b8698a0f | 395 | const struct common_sched_info_def haifa_common_sched_info = |
e855c69d AB |
396 | { |
397 | NULL, /* fix_recovery_cfg */ | |
398 | NULL, /* add_block */ | |
399 | NULL, /* estimate_number_of_insns */ | |
400 | haifa_luid_for_non_insn, /* luid_for_non_insn */ | |
401 | SCHED_PASS_UNKNOWN /* sched_pass_id */ | |
402 | }; | |
403 | ||
e855c69d | 404 | /* Mapping from instruction UID to its Logical UID. */ |
7de76362 | 405 | vec<int> sched_luids; |
e855c69d AB |
406 | |
407 | /* Next LUID to assign to an instruction. */ | |
408 | int sched_max_luid = 1; | |
409 | ||
410 | /* Haifa Instruction Data. */ | |
7de76362 | 411 | vec<haifa_insn_data_def> h_i_d; |
e855c69d AB |
412 | |
413 | void (* sched_init_only_bb) (basic_block, basic_block); | |
414 | ||
415 | /* Split block function. Different schedulers might use different functions | |
416 | to handle their internal data consistent. */ | |
417 | basic_block (* sched_split_block) (basic_block, rtx); | |
418 | ||
419 | /* Create empty basic block after the specified block. */ | |
420 | basic_block (* sched_create_empty_bb) (basic_block); | |
421 | ||
7a1aca9c RS |
422 | /* Return the number of cycles until INSN is expected to be ready. |
423 | Return zero if it already is. */ | |
424 | static int | |
f57aa6b0 | 425 | insn_delay (rtx_insn *insn) |
7a1aca9c RS |
426 | { |
427 | return MAX (INSN_TICK (insn) - clock_var, 0); | |
428 | } | |
429 | ||
15aab9c0 | 430 | static int |
9678086d | 431 | may_trap_exp (const_rtx x, int is_store) |
15aab9c0 VM |
432 | { |
433 | enum rtx_code code; | |
434 | ||
435 | if (x == 0) | |
436 | return TRAP_FREE; | |
437 | code = GET_CODE (x); | |
438 | if (is_store) | |
439 | { | |
440 | if (code == MEM && may_trap_p (x)) | |
441 | return TRAP_RISKY; | |
442 | else | |
443 | return TRAP_FREE; | |
444 | } | |
445 | if (code == MEM) | |
446 | { | |
447 | /* The insn uses memory: a volatile load. */ | |
448 | if (MEM_VOLATILE_P (x)) | |
449 | return IRISKY; | |
450 | /* An exception-free load. */ | |
451 | if (!may_trap_p (x)) | |
452 | return IFREE; | |
453 | /* A load with 1 base register, to be further checked. */ | |
454 | if (CONST_BASED_ADDRESS_P (XEXP (x, 0))) | |
455 | return PFREE_CANDIDATE; | |
456 | /* No info on the load, to be further checked. */ | |
457 | return PRISKY_CANDIDATE; | |
458 | } | |
459 | else | |
460 | { | |
461 | const char *fmt; | |
462 | int i, insn_class = TRAP_FREE; | |
463 | ||
464 | /* Neither store nor load, check if it may cause a trap. */ | |
465 | if (may_trap_p (x)) | |
466 | return TRAP_RISKY; | |
467 | /* Recursive step: walk the insn... */ | |
468 | fmt = GET_RTX_FORMAT (code); | |
469 | for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--) | |
470 | { | |
471 | if (fmt[i] == 'e') | |
472 | { | |
473 | int tmp_class = may_trap_exp (XEXP (x, i), is_store); | |
474 | insn_class = WORST_CLASS (insn_class, tmp_class); | |
475 | } | |
476 | else if (fmt[i] == 'E') | |
477 | { | |
478 | int j; | |
479 | for (j = 0; j < XVECLEN (x, i); j++) | |
480 | { | |
481 | int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store); | |
482 | insn_class = WORST_CLASS (insn_class, tmp_class); | |
483 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
484 | break; | |
485 | } | |
486 | } | |
487 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
488 | break; | |
489 | } | |
490 | return insn_class; | |
491 | } | |
492 | } | |
493 | ||
ac4a7e21 AM |
494 | /* Classifies rtx X of an insn for the purpose of verifying that X can be |
495 | executed speculatively (and consequently the insn can be moved | |
496 | speculatively), by examining X, returning: | |
15aab9c0 VM |
497 | TRAP_RISKY: store, or risky non-load insn (e.g. division by variable). |
498 | TRAP_FREE: non-load insn. | |
a98ebe2e | 499 | IFREE: load from a globally safe location. |
15aab9c0 VM |
500 | IRISKY: volatile load. |
501 | PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for | |
502 | being either PFREE or PRISKY. */ | |
503 | ||
ac4a7e21 AM |
504 | static int |
505 | haifa_classify_rtx (const_rtx x) | |
15aab9c0 | 506 | { |
15aab9c0 VM |
507 | int tmp_class = TRAP_FREE; |
508 | int insn_class = TRAP_FREE; | |
509 | enum rtx_code code; | |
510 | ||
ac4a7e21 | 511 | if (GET_CODE (x) == PARALLEL) |
15aab9c0 | 512 | { |
ac4a7e21 | 513 | int i, len = XVECLEN (x, 0); |
15aab9c0 VM |
514 | |
515 | for (i = len - 1; i >= 0; i--) | |
516 | { | |
ac4a7e21 | 517 | tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i)); |
15aab9c0 VM |
518 | insn_class = WORST_CLASS (insn_class, tmp_class); |
519 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
520 | break; | |
521 | } | |
522 | } | |
523 | else | |
524 | { | |
ac4a7e21 | 525 | code = GET_CODE (x); |
15aab9c0 VM |
526 | switch (code) |
527 | { | |
528 | case CLOBBER: | |
529 | /* Test if it is a 'store'. */ | |
ac4a7e21 | 530 | tmp_class = may_trap_exp (XEXP (x, 0), 1); |
15aab9c0 | 531 | break; |
8df47bdf AH |
532 | case CLOBBER_HIGH: |
533 | gcc_assert (REG_P (XEXP (x, 0))); | |
534 | break; | |
15aab9c0 VM |
535 | case SET: |
536 | /* Test if it is a store. */ | |
ac4a7e21 | 537 | tmp_class = may_trap_exp (SET_DEST (x), 1); |
15aab9c0 VM |
538 | if (tmp_class == TRAP_RISKY) |
539 | break; | |
540 | /* Test if it is a load. */ | |
541 | tmp_class = | |
542 | WORST_CLASS (tmp_class, | |
ac4a7e21 | 543 | may_trap_exp (SET_SRC (x), 0)); |
15aab9c0 VM |
544 | break; |
545 | case COND_EXEC: | |
ac4a7e21 AM |
546 | tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x)); |
547 | if (tmp_class == TRAP_RISKY) | |
548 | break; | |
549 | tmp_class = WORST_CLASS (tmp_class, | |
550 | may_trap_exp (COND_EXEC_TEST (x), 0)); | |
551 | break; | |
15aab9c0 VM |
552 | case TRAP_IF: |
553 | tmp_class = TRAP_RISKY; | |
554 | break; | |
555 | default:; | |
556 | } | |
557 | insn_class = tmp_class; | |
558 | } | |
559 | ||
560 | return insn_class; | |
561 | } | |
562 | ||
ac4a7e21 AM |
563 | int |
564 | haifa_classify_insn (const_rtx insn) | |
565 | { | |
566 | return haifa_classify_rtx (PATTERN (insn)); | |
567 | } | |
06d7e8e7 BS |
568 | \f |
569 | /* After the scheduler initialization function has been called, this function | |
570 | can be called to enable modulo scheduling. II is the initiation interval | |
571 | we should use, it affects the delays for delay_pairs that were recorded as | |
572 | separated by a given number of stages. | |
573 | ||
574 | MAX_STAGES provides us with a limit | |
575 | after which we give up scheduling; the caller must have unrolled at least | |
576 | as many copies of the loop body and recorded delay_pairs for them. | |
577 | ||
578 | INSNS is the number of real (non-debug) insns in one iteration of | |
579 | the loop. MAX_UID can be used to test whether an insn belongs to | |
580 | the first iteration of the loop; all of them have a uid lower than | |
581 | MAX_UID. */ | |
582 | void | |
583 | set_modulo_params (int ii, int max_stages, int insns, int max_uid) | |
584 | { | |
585 | modulo_ii = ii; | |
586 | modulo_max_stages = max_stages; | |
587 | modulo_n_insns = insns; | |
588 | modulo_iter0_max_uid = max_uid; | |
589 | modulo_backtracks_left = PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS); | |
590 | } | |
ac4a7e21 | 591 | |
26965010 BS |
592 | /* A structure to record a pair of insns where the first one is a real |
593 | insn that has delay slots, and the second is its delayed shadow. | |
594 | I1 is scheduled normally and will emit an assembly instruction, | |
595 | while I2 describes the side effect that takes place at the | |
596 | transition between cycles CYCLES and (CYCLES + 1) after I1. */ | |
597 | struct delay_pair | |
598 | { | |
599 | struct delay_pair *next_same_i1; | |
ce1ce33a | 600 | rtx_insn *i1, *i2; |
26965010 | 601 | int cycles; |
06d7e8e7 BS |
602 | /* When doing modulo scheduling, we a delay_pair can also be used to |
603 | show that I1 and I2 are the same insn in a different stage. If that | |
604 | is the case, STAGES will be nonzero. */ | |
605 | int stages; | |
26965010 BS |
606 | }; |
607 | ||
4a8fb1a1 LC |
608 | /* Helpers for delay hashing. */ |
609 | ||
8d67ee55 | 610 | struct delay_i1_hasher : nofree_ptr_hash <delay_pair> |
4a8fb1a1 | 611 | { |
67f58944 TS |
612 | typedef void *compare_type; |
613 | static inline hashval_t hash (const delay_pair *); | |
614 | static inline bool equal (const delay_pair *, const void *); | |
4a8fb1a1 LC |
615 | }; |
616 | ||
617 | /* Returns a hash value for X, based on hashing just I1. */ | |
618 | ||
619 | inline hashval_t | |
67f58944 | 620 | delay_i1_hasher::hash (const delay_pair *x) |
4a8fb1a1 LC |
621 | { |
622 | return htab_hash_pointer (x->i1); | |
623 | } | |
624 | ||
625 | /* Return true if I1 of pair X is the same as that of pair Y. */ | |
626 | ||
627 | inline bool | |
67f58944 | 628 | delay_i1_hasher::equal (const delay_pair *x, const void *y) |
4a8fb1a1 LC |
629 | { |
630 | return x->i1 == y; | |
631 | } | |
632 | ||
95fbe13e | 633 | struct delay_i2_hasher : free_ptr_hash <delay_pair> |
4a8fb1a1 | 634 | { |
67f58944 TS |
635 | typedef void *compare_type; |
636 | static inline hashval_t hash (const delay_pair *); | |
637 | static inline bool equal (const delay_pair *, const void *); | |
4a8fb1a1 LC |
638 | }; |
639 | ||
640 | /* Returns a hash value for X, based on hashing just I2. */ | |
641 | ||
642 | inline hashval_t | |
67f58944 | 643 | delay_i2_hasher::hash (const delay_pair *x) |
4a8fb1a1 LC |
644 | { |
645 | return htab_hash_pointer (x->i2); | |
646 | } | |
647 | ||
648 | /* Return true if I2 of pair X is the same as that of pair Y. */ | |
649 | ||
650 | inline bool | |
67f58944 | 651 | delay_i2_hasher::equal (const delay_pair *x, const void *y) |
4a8fb1a1 LC |
652 | { |
653 | return x->i2 == y; | |
654 | } | |
655 | ||
26965010 BS |
656 | /* Two hash tables to record delay_pairs, one indexed by I1 and the other |
657 | indexed by I2. */ | |
c203e8a7 TS |
658 | static hash_table<delay_i1_hasher> *delay_htab; |
659 | static hash_table<delay_i2_hasher> *delay_htab_i2; | |
26965010 | 660 | |
06d7e8e7 BS |
661 | /* Called through htab_traverse. Walk the hashtable using I2 as |
662 | index, and delete all elements involving an UID higher than | |
663 | that pointed to by *DATA. */ | |
4a8fb1a1 LC |
664 | int |
665 | haifa_htab_i2_traverse (delay_pair **slot, int *data) | |
06d7e8e7 | 666 | { |
4a8fb1a1 LC |
667 | int maxuid = *data; |
668 | struct delay_pair *p = *slot; | |
06d7e8e7 BS |
669 | if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid) |
670 | { | |
c203e8a7 | 671 | delay_htab_i2->clear_slot (slot); |
06d7e8e7 BS |
672 | } |
673 | return 1; | |
674 | } | |
675 | ||
676 | /* Called through htab_traverse. Walk the hashtable using I2 as | |
677 | index, and delete all elements involving an UID higher than | |
678 | that pointed to by *DATA. */ | |
4a8fb1a1 LC |
679 | int |
680 | haifa_htab_i1_traverse (delay_pair **pslot, int *data) | |
06d7e8e7 | 681 | { |
4a8fb1a1 | 682 | int maxuid = *data; |
06d7e8e7 BS |
683 | struct delay_pair *p, *first, **pprev; |
684 | ||
685 | if (INSN_UID ((*pslot)->i1) >= maxuid) | |
686 | { | |
c203e8a7 | 687 | delay_htab->clear_slot (pslot); |
06d7e8e7 BS |
688 | return 1; |
689 | } | |
690 | pprev = &first; | |
691 | for (p = *pslot; p; p = p->next_same_i1) | |
692 | { | |
693 | if (INSN_UID (p->i2) < maxuid) | |
694 | { | |
695 | *pprev = p; | |
696 | pprev = &p->next_same_i1; | |
697 | } | |
698 | } | |
699 | *pprev = NULL; | |
700 | if (first == NULL) | |
c203e8a7 | 701 | delay_htab->clear_slot (pslot); |
06d7e8e7 BS |
702 | else |
703 | *pslot = first; | |
704 | return 1; | |
705 | } | |
706 | ||
707 | /* Discard all delay pairs which involve an insn with an UID higher | |
708 | than MAX_UID. */ | |
709 | void | |
710 | discard_delay_pairs_above (int max_uid) | |
711 | { | |
c203e8a7 TS |
712 | delay_htab->traverse <int *, haifa_htab_i1_traverse> (&max_uid); |
713 | delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (&max_uid); | |
26965010 BS |
714 | } |
715 | ||
06d7e8e7 BS |
716 | /* This function can be called by a port just before it starts the final |
717 | scheduling pass. It records the fact that an instruction with delay | |
718 | slots has been split into two insns, I1 and I2. The first one will be | |
719 | scheduled normally and initiates the operation. The second one is a | |
720 | shadow which must follow a specific number of cycles after I1; its only | |
721 | purpose is to show the side effect that occurs at that cycle in the RTL. | |
722 | If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN, | |
723 | while I2 retains the original insn type. | |
724 | ||
725 | There are two ways in which the number of cycles can be specified, | |
726 | involving the CYCLES and STAGES arguments to this function. If STAGES | |
727 | is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor | |
728 | which is multiplied by MODULO_II to give the number of cycles. This is | |
729 | only useful if the caller also calls set_modulo_params to enable modulo | |
730 | scheduling. */ | |
26965010 BS |
731 | |
732 | void | |
ce1ce33a | 733 | record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages) |
26965010 BS |
734 | { |
735 | struct delay_pair *p = XNEW (struct delay_pair); | |
736 | struct delay_pair **slot; | |
737 | ||
738 | p->i1 = i1; | |
739 | p->i2 = i2; | |
740 | p->cycles = cycles; | |
06d7e8e7 | 741 | p->stages = stages; |
26965010 | 742 | |
c203e8a7 | 743 | if (!delay_htab) |
26965010 | 744 | { |
c203e8a7 TS |
745 | delay_htab = new hash_table<delay_i1_hasher> (10); |
746 | delay_htab_i2 = new hash_table<delay_i2_hasher> (10); | |
26965010 | 747 | } |
c203e8a7 | 748 | slot = delay_htab->find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT); |
26965010 BS |
749 | p->next_same_i1 = *slot; |
750 | *slot = p; | |
c203e8a7 | 751 | slot = delay_htab_i2->find_slot (p, INSERT); |
26965010 BS |
752 | *slot = p; |
753 | } | |
754 | ||
e2724e63 BS |
755 | /* Examine the delay pair hashtable to see if INSN is a shadow for another, |
756 | and return the other insn if so. Return NULL otherwise. */ | |
f57aa6b0 DM |
757 | rtx_insn * |
758 | real_insn_for_shadow (rtx_insn *insn) | |
e2724e63 BS |
759 | { |
760 | struct delay_pair *pair; | |
761 | ||
c203e8a7 | 762 | if (!delay_htab) |
f57aa6b0 | 763 | return NULL; |
e2724e63 | 764 | |
c203e8a7 | 765 | pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn)); |
e2724e63 | 766 | if (!pair || pair->stages > 0) |
f57aa6b0 | 767 | return NULL; |
e2724e63 BS |
768 | return pair->i1; |
769 | } | |
770 | ||
26965010 BS |
771 | /* For a pair P of insns, return the fixed distance in cycles from the first |
772 | insn after which the second must be scheduled. */ | |
773 | static int | |
774 | pair_delay (struct delay_pair *p) | |
775 | { | |
06d7e8e7 BS |
776 | if (p->stages == 0) |
777 | return p->cycles; | |
778 | else | |
779 | return p->stages * modulo_ii; | |
26965010 BS |
780 | } |
781 | ||
782 | /* Given an insn INSN, add a dependence on its delayed shadow if it | |
783 | has one. Also try to find situations where shadows depend on each other | |
784 | and add dependencies to the real insns to limit the amount of backtracking | |
785 | needed. */ | |
786 | void | |
ce1ce33a | 787 | add_delay_dependencies (rtx_insn *insn) |
26965010 BS |
788 | { |
789 | struct delay_pair *pair; | |
790 | sd_iterator_def sd_it; | |
791 | dep_t dep; | |
792 | ||
c203e8a7 | 793 | if (!delay_htab) |
26965010 BS |
794 | return; |
795 | ||
c203e8a7 | 796 | pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn)); |
26965010 BS |
797 | if (!pair) |
798 | return; | |
799 | add_dependence (insn, pair->i1, REG_DEP_ANTI); | |
06d7e8e7 BS |
800 | if (pair->stages) |
801 | return; | |
26965010 BS |
802 | |
803 | FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep) | |
804 | { | |
9b517712 | 805 | rtx_insn *pro = DEP_PRO (dep); |
26965010 | 806 | struct delay_pair *other_pair |
c203e8a7 | 807 | = delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro)); |
06d7e8e7 | 808 | if (!other_pair || other_pair->stages) |
26965010 BS |
809 | continue; |
810 | if (pair_delay (other_pair) >= pair_delay (pair)) | |
811 | { | |
812 | if (sched_verbose >= 4) | |
813 | { | |
814 | fprintf (sched_dump, ";;\tadding dependence %d <- %d\n", | |
815 | INSN_UID (other_pair->i1), | |
816 | INSN_UID (pair->i1)); | |
817 | fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n", | |
818 | INSN_UID (pair->i1), | |
819 | INSN_UID (pair->i2), | |
820 | pair_delay (pair)); | |
821 | fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n", | |
822 | INSN_UID (other_pair->i1), | |
823 | INSN_UID (other_pair->i2), | |
824 | pair_delay (other_pair)); | |
825 | } | |
826 | add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI); | |
827 | } | |
828 | } | |
829 | } | |
830 | \f | |
8c660648 | 831 | /* Forward declarations. */ |
fae15c93 | 832 | |
079c81d0 | 833 | static int priority (rtx_insn *, bool force_recompute = false); |
340c7904 | 834 | static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *); |
1d088dee | 835 | static int rank_for_schedule (const void *, const void *); |
ce1ce33a DM |
836 | static void swap_sort (rtx_insn **, int); |
837 | static void queue_insn (rtx_insn *, int, const char *); | |
838 | static int schedule_insn (rtx_insn *); | |
f57aa6b0 | 839 | static void adjust_priority (rtx_insn *); |
1d088dee | 840 | static void advance_one_cycle (void); |
e855c69d AB |
841 | static void extend_h_i_d (void); |
842 | ||
8c660648 | 843 | |
8c660648 JL |
844 | /* Notes handling mechanism: |
845 | ========================= | |
846 | Generally, NOTES are saved before scheduling and restored after scheduling. | |
07c02828 | 847 | The scheduler distinguishes between two types of notes: |
8c660648 | 848 | |
6039a0c7 | 849 | (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes: |
8c660648 JL |
850 | Before scheduling a region, a pointer to the note is added to the insn |
851 | that follows or precedes it. (This happens as part of the data dependence | |
852 | computation). After scheduling an insn, the pointer contained in it is | |
853 | used for regenerating the corresponding note (in reemit_notes). | |
854 | ||
6039a0c7 | 855 | (2) All other notes (e.g. INSN_DELETED): Before scheduling a block, |
8c660648 JL |
856 | these notes are put in a list (in rm_other_notes() and |
857 | unlink_other_notes ()). After scheduling the block, these notes are | |
858 | inserted at the beginning of the block (in schedule_block()). */ | |
859 | ||
ce1ce33a DM |
860 | static void ready_add (struct ready_list *, rtx_insn *, bool); |
861 | static rtx_insn *ready_remove_first (struct ready_list *); | |
862 | static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready); | |
3fe41456 | 863 | |
1d088dee | 864 | static void queue_to_ready (struct ready_list *); |
569fa502 | 865 | static int early_queue_to_ready (state_t, struct ready_list *); |
176f9a7b | 866 | |
fae15c93 | 867 | /* The following functions are used to implement multi-pass scheduling |
fa0aee89 | 868 | on the first cycle. */ |
ce1ce33a | 869 | static rtx_insn *ready_remove (struct ready_list *, int); |
90831096 | 870 | static void ready_remove_insn (rtx_insn *); |
fae15c93 | 871 | |
f57aa6b0 | 872 | static void fix_inter_tick (rtx_insn *, rtx_insn *); |
ce1ce33a DM |
873 | static int fix_tick_ready (rtx_insn *); |
874 | static void change_queue_index (rtx_insn *, int); | |
63f54b1a | 875 | |
496d7bb0 MK |
876 | /* The following functions are used to implement scheduling of data/control |
877 | speculative instructions. */ | |
878 | ||
879 | static void extend_h_i_d (void); | |
f57aa6b0 DM |
880 | static void init_h_i_d (rtx_insn *); |
881 | static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *); | |
ce1ce33a | 882 | static void generate_recovery_code (rtx_insn *); |
90831096 | 883 | static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t); |
ce1ce33a | 884 | static void begin_speculative_block (rtx_insn *); |
6144a836 | 885 | static void add_to_speculative_block (rtx_insn *); |
e855c69d | 886 | static void init_before_recovery (basic_block *); |
ce1ce33a | 887 | static void create_check_block_twin (rtx_insn *, bool); |
496d7bb0 | 888 | static void fix_recovery_deps (basic_block); |
f57aa6b0 DM |
889 | static bool haifa_change_pattern (rtx_insn *, rtx); |
890 | static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *); | |
496d7bb0 | 891 | static void restore_bb_notes (basic_block); |
f57aa6b0 DM |
892 | static void fix_jump_move (rtx_insn *); |
893 | static void move_block_after_check (rtx_insn *); | |
9771b263 | 894 | static void move_succs (vec<edge, va_gc> **, basic_block); |
ce1ce33a | 895 | static void sched_remove_insn (rtx_insn *); |
6144a836 | 896 | static void clear_priorities (rtx_insn *, rtx_vec_t *); |
916fa4f0 | 897 | static void calc_priorities (rtx_vec_t); |
0e0ce50d | 898 | static void add_jump_dependencies (rtx_insn *, rtx_insn *); |
496d7bb0 | 899 | |
8c660648 JL |
900 | #endif /* INSN_SCHEDULING */ |
901 | \f | |
1708fd40 | 902 | /* Point to state used for the current scheduling pass. */ |
e855c69d | 903 | struct haifa_sched_info *current_sched_info; |
8c660648 JL |
904 | \f |
905 | #ifndef INSN_SCHEDULING | |
906 | void | |
10d22567 | 907 | schedule_insns (void) |
8c660648 JL |
908 | { |
909 | } | |
910 | #else | |
cbb13457 | 911 | |
ce18efcb VM |
912 | /* Do register pressure sensitive insn scheduling if the flag is set |
913 | up. */ | |
60867e8c | 914 | enum sched_pressure_algorithm sched_pressure; |
ce18efcb | 915 | |
1756cb66 | 916 | /* Map regno -> its pressure class. The map defined only when |
7a1aca9c | 917 | SCHED_PRESSURE != SCHED_PRESSURE_NONE. */ |
1756cb66 | 918 | enum reg_class *sched_regno_pressure_class; |
ce18efcb | 919 | |
1756cb66 | 920 | /* The current register pressure. Only elements corresponding pressure |
ce18efcb VM |
921 | classes are defined. */ |
922 | static int curr_reg_pressure[N_REG_CLASSES]; | |
923 | ||
924 | /* Saved value of the previous array. */ | |
925 | static int saved_reg_pressure[N_REG_CLASSES]; | |
926 | ||
927 | /* Register living at given scheduling point. */ | |
928 | static bitmap curr_reg_live; | |
929 | ||
930 | /* Saved value of the previous array. */ | |
931 | static bitmap saved_reg_live; | |
932 | ||
933 | /* Registers mentioned in the current region. */ | |
934 | static bitmap region_ref_regs; | |
935 | ||
f380f608 RS |
936 | /* Temporary bitmap used for SCHED_PRESSURE_MODEL. */ |
937 | static bitmap tmp_bitmap; | |
938 | ||
4dd9ac6c MK |
939 | /* Effective number of available registers of a given class (see comment |
940 | in sched_pressure_start_bb). */ | |
941 | static int sched_class_regs_num[N_REG_CLASSES]; | |
6d5c37d0 | 942 | /* Number of call_saved_regs and fixed_regs. Helpers for calculating of |
4dd9ac6c | 943 | sched_class_regs_num. */ |
6d5c37d0 PH |
944 | static int call_saved_regs_num[N_REG_CLASSES]; |
945 | static int fixed_regs_num[N_REG_CLASSES]; | |
4dd9ac6c | 946 | |
ce18efcb VM |
947 | /* Initiate register pressure relative info for scheduling the current |
948 | region. Currently it is only clearing register mentioned in the | |
949 | current region. */ | |
950 | void | |
951 | sched_init_region_reg_pressure_info (void) | |
952 | { | |
953 | bitmap_clear (region_ref_regs); | |
954 | } | |
955 | ||
7a1aca9c RS |
956 | /* PRESSURE[CL] describes the pressure on register class CL. Update it |
957 | for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO. | |
958 | LIVE tracks the set of live registers; if it is null, assume that | |
959 | every birth or death is genuine. */ | |
960 | static inline void | |
961 | mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p) | |
ce18efcb | 962 | { |
1756cb66 | 963 | enum reg_class pressure_class; |
ce18efcb | 964 | |
1756cb66 | 965 | pressure_class = sched_regno_pressure_class[regno]; |
ce18efcb VM |
966 | if (regno >= FIRST_PSEUDO_REGISTER) |
967 | { | |
1756cb66 | 968 | if (pressure_class != NO_REGS) |
ce18efcb VM |
969 | { |
970 | if (birth_p) | |
971 | { | |
7a1aca9c RS |
972 | if (!live || bitmap_set_bit (live, regno)) |
973 | pressure[pressure_class] | |
974 | += (ira_reg_class_max_nregs | |
975 | [pressure_class][PSEUDO_REGNO_MODE (regno)]); | |
ce18efcb VM |
976 | } |
977 | else | |
978 | { | |
7a1aca9c RS |
979 | if (!live || bitmap_clear_bit (live, regno)) |
980 | pressure[pressure_class] | |
981 | -= (ira_reg_class_max_nregs | |
982 | [pressure_class][PSEUDO_REGNO_MODE (regno)]); | |
ce18efcb VM |
983 | } |
984 | } | |
985 | } | |
1756cb66 | 986 | else if (pressure_class != NO_REGS |
ce18efcb VM |
987 | && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno)) |
988 | { | |
989 | if (birth_p) | |
990 | { | |
7a1aca9c RS |
991 | if (!live || bitmap_set_bit (live, regno)) |
992 | pressure[pressure_class]++; | |
ce18efcb VM |
993 | } |
994 | else | |
995 | { | |
7a1aca9c RS |
996 | if (!live || bitmap_clear_bit (live, regno)) |
997 | pressure[pressure_class]--; | |
ce18efcb VM |
998 | } |
999 | } | |
1000 | } | |
1001 | ||
1002 | /* Initiate current register pressure related info from living | |
1003 | registers given by LIVE. */ | |
1004 | static void | |
1005 | initiate_reg_pressure_info (bitmap live) | |
1006 | { | |
1007 | int i; | |
1008 | unsigned int j; | |
1009 | bitmap_iterator bi; | |
1010 | ||
1756cb66 VM |
1011 | for (i = 0; i < ira_pressure_classes_num; i++) |
1012 | curr_reg_pressure[ira_pressure_classes[i]] = 0; | |
ce18efcb VM |
1013 | bitmap_clear (curr_reg_live); |
1014 | EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi) | |
7a1aca9c RS |
1015 | if (sched_pressure == SCHED_PRESSURE_MODEL |
1016 | || current_nr_blocks == 1 | |
1017 | || bitmap_bit_p (region_ref_regs, j)) | |
1018 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true); | |
ce18efcb VM |
1019 | } |
1020 | ||
1021 | /* Mark registers in X as mentioned in the current region. */ | |
1022 | static void | |
1023 | setup_ref_regs (rtx x) | |
1024 | { | |
07a737f3 | 1025 | int i, j; |
ce18efcb VM |
1026 | const RTX_CODE code = GET_CODE (x); |
1027 | const char *fmt; | |
1028 | ||
1029 | if (REG_P (x)) | |
1030 | { | |
07a737f3 | 1031 | bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x)); |
ce18efcb VM |
1032 | return; |
1033 | } | |
1034 | fmt = GET_RTX_FORMAT (code); | |
1035 | for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--) | |
1036 | if (fmt[i] == 'e') | |
1037 | setup_ref_regs (XEXP (x, i)); | |
1038 | else if (fmt[i] == 'E') | |
1039 | { | |
1040 | for (j = 0; j < XVECLEN (x, i); j++) | |
1041 | setup_ref_regs (XVECEXP (x, i, j)); | |
1042 | } | |
1043 | } | |
1044 | ||
1045 | /* Initiate current register pressure related info at the start of | |
1046 | basic block BB. */ | |
1047 | static void | |
1048 | initiate_bb_reg_pressure_info (basic_block bb) | |
1049 | { | |
6ab31ade | 1050 | unsigned int i ATTRIBUTE_UNUSED; |
dc01c3d1 | 1051 | rtx_insn *insn; |
ce18efcb VM |
1052 | |
1053 | if (current_nr_blocks > 1) | |
1054 | FOR_BB_INSNS (bb, insn) | |
69b45d71 | 1055 | if (NONDEBUG_INSN_P (insn)) |
ce18efcb VM |
1056 | setup_ref_regs (PATTERN (insn)); |
1057 | initiate_reg_pressure_info (df_get_live_in (bb)); | |
ce18efcb VM |
1058 | if (bb_has_eh_pred (bb)) |
1059 | for (i = 0; ; ++i) | |
1060 | { | |
1061 | unsigned int regno = EH_RETURN_DATA_REGNO (i); | |
b8698a0f | 1062 | |
ce18efcb VM |
1063 | if (regno == INVALID_REGNUM) |
1064 | break; | |
1065 | if (! bitmap_bit_p (df_get_live_in (bb), regno)) | |
7a1aca9c RS |
1066 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, |
1067 | regno, true); | |
ce18efcb | 1068 | } |
ce18efcb VM |
1069 | } |
1070 | ||
1071 | /* Save current register pressure related info. */ | |
1072 | static void | |
1073 | save_reg_pressure (void) | |
1074 | { | |
1075 | int i; | |
b8698a0f | 1076 | |
1756cb66 VM |
1077 | for (i = 0; i < ira_pressure_classes_num; i++) |
1078 | saved_reg_pressure[ira_pressure_classes[i]] | |
1079 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb VM |
1080 | bitmap_copy (saved_reg_live, curr_reg_live); |
1081 | } | |
1082 | ||
1083 | /* Restore saved register pressure related info. */ | |
1084 | static void | |
1085 | restore_reg_pressure (void) | |
1086 | { | |
1087 | int i; | |
b8698a0f | 1088 | |
1756cb66 VM |
1089 | for (i = 0; i < ira_pressure_classes_num; i++) |
1090 | curr_reg_pressure[ira_pressure_classes[i]] | |
1091 | = saved_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb VM |
1092 | bitmap_copy (curr_reg_live, saved_reg_live); |
1093 | } | |
1094 | ||
1095 | /* Return TRUE if the register is dying after its USE. */ | |
1096 | static bool | |
1097 | dying_use_p (struct reg_use_data *use) | |
1098 | { | |
1099 | struct reg_use_data *next; | |
1100 | ||
1101 | for (next = use->next_regno_use; next != use; next = next->next_regno_use) | |
bb956f3e AO |
1102 | if (NONDEBUG_INSN_P (next->insn) |
1103 | && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED) | |
ce18efcb VM |
1104 | return false; |
1105 | return true; | |
1106 | } | |
1107 | ||
1108 | /* Print info about the current register pressure and its excess for | |
1756cb66 | 1109 | each pressure class. */ |
ce18efcb VM |
1110 | static void |
1111 | print_curr_reg_pressure (void) | |
1112 | { | |
1113 | int i; | |
1114 | enum reg_class cl; | |
1115 | ||
1116 | fprintf (sched_dump, ";;\t"); | |
1756cb66 | 1117 | for (i = 0; i < ira_pressure_classes_num; i++) |
ce18efcb | 1118 | { |
1756cb66 | 1119 | cl = ira_pressure_classes[i]; |
ce18efcb VM |
1120 | gcc_assert (curr_reg_pressure[cl] >= 0); |
1121 | fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl], | |
1122 | curr_reg_pressure[cl], | |
4dd9ac6c | 1123 | curr_reg_pressure[cl] - sched_class_regs_num[cl]); |
ce18efcb VM |
1124 | } |
1125 | fprintf (sched_dump, "\n"); | |
1126 | } | |
e2724e63 BS |
1127 | \f |
1128 | /* Determine if INSN has a condition that is clobbered if a register | |
1129 | in SET_REGS is modified. */ | |
1130 | static bool | |
ce1ce33a | 1131 | cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs) |
e2724e63 BS |
1132 | { |
1133 | rtx pat = PATTERN (insn); | |
1134 | gcc_assert (GET_CODE (pat) == COND_EXEC); | |
1135 | if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0)))) | |
1136 | { | |
1137 | sd_iterator_def sd_it; | |
1138 | dep_t dep; | |
1139 | haifa_change_pattern (insn, ORIG_PAT (insn)); | |
1140 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
1141 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
1142 | TODO_SPEC (insn) = HARD_DEP; | |
1143 | if (sched_verbose >= 2) | |
1144 | fprintf (sched_dump, | |
1145 | ";;\t\tdequeue insn %s because of clobbered condition\n", | |
1146 | (*current_sched_info->print_insn) (insn, 0)); | |
1147 | return true; | |
1148 | } | |
1149 | ||
1150 | return false; | |
1151 | } | |
1152 | ||
1a83e602 BS |
1153 | /* This function should be called after modifying the pattern of INSN, |
1154 | to update scheduler data structures as needed. */ | |
1155 | static void | |
f57aa6b0 | 1156 | update_insn_after_change (rtx_insn *insn) |
1a83e602 BS |
1157 | { |
1158 | sd_iterator_def sd_it; | |
1159 | dep_t dep; | |
1160 | ||
1161 | dfa_clear_single_insn_cache (insn); | |
1162 | ||
1163 | sd_it = sd_iterator_start (insn, | |
1164 | SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK); | |
1165 | while (sd_iterator_cond (&sd_it, &dep)) | |
1166 | { | |
1167 | DEP_COST (dep) = UNKNOWN_DEP_COST; | |
1168 | sd_iterator_next (&sd_it); | |
1169 | } | |
1170 | ||
1171 | /* Invalidate INSN_COST, so it'll be recalculated. */ | |
1172 | INSN_COST (insn) = -1; | |
1173 | /* Invalidate INSN_TICK, so it'll be recalculated. */ | |
1174 | INSN_TICK (insn) = INVALID_TICK; | |
340c7904 MK |
1175 | |
1176 | /* Invalidate autoprefetch data entry. */ | |
1177 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status | |
1178 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
1179 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status | |
1180 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
1a83e602 BS |
1181 | } |
1182 | ||
1a83e602 BS |
1183 | |
1184 | /* Two VECs, one to hold dependencies for which pattern replacements | |
1185 | need to be applied or restored at the start of the next cycle, and | |
1186 | another to hold an integer that is either one, to apply the | |
1187 | corresponding replacement, or zero to restore it. */ | |
9771b263 DN |
1188 | static vec<dep_t> next_cycle_replace_deps; |
1189 | static vec<int> next_cycle_apply; | |
1a83e602 BS |
1190 | |
1191 | static void apply_replacement (dep_t, bool); | |
1192 | static void restore_pattern (dep_t, bool); | |
1193 | ||
e2724e63 BS |
1194 | /* Look at the remaining dependencies for insn NEXT, and compute and return |
1195 | the TODO_SPEC value we should use for it. This is called after one of | |
1a83e602 BS |
1196 | NEXT's dependencies has been resolved. |
1197 | We also perform pattern replacements for predication, and for broken | |
1198 | replacement dependencies. The latter is only done if FOR_BACKTRACK is | |
1199 | false. */ | |
ce18efcb | 1200 | |
e2724e63 | 1201 | static ds_t |
f57aa6b0 | 1202 | recompute_todo_spec (rtx_insn *next, bool for_backtrack) |
e2724e63 BS |
1203 | { |
1204 | ds_t new_ds; | |
1205 | sd_iterator_def sd_it; | |
1a83e602 | 1206 | dep_t dep, modify_dep = NULL; |
e2724e63 BS |
1207 | int n_spec = 0; |
1208 | int n_control = 0; | |
1a83e602 | 1209 | int n_replace = 0; |
e2724e63 BS |
1210 | bool first_p = true; |
1211 | ||
1212 | if (sd_lists_empty_p (next, SD_LIST_BACK)) | |
1213 | /* NEXT has all its dependencies resolved. */ | |
1214 | return 0; | |
1215 | ||
1216 | if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK)) | |
1217 | return HARD_DEP; | |
1218 | ||
61a17dca JG |
1219 | /* If NEXT is intended to sit adjacent to this instruction, we don't |
1220 | want to try to break any dependencies. Treat it as a HARD_DEP. */ | |
1221 | if (SCHED_GROUP_P (next)) | |
1222 | return HARD_DEP; | |
1223 | ||
e2724e63 BS |
1224 | /* Now we've got NEXT with speculative deps only. |
1225 | 1. Look at the deps to see what we have to do. | |
1226 | 2. Check if we can do 'todo'. */ | |
1227 | new_ds = 0; | |
1228 | ||
1229 | FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep) | |
1230 | { | |
9b517712 | 1231 | rtx_insn *pro = DEP_PRO (dep); |
e2724e63 BS |
1232 | ds_t ds = DEP_STATUS (dep) & SPECULATIVE; |
1233 | ||
1a83e602 | 1234 | if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next)) |
e2724e63 BS |
1235 | continue; |
1236 | ||
1237 | if (ds) | |
1238 | { | |
1239 | n_spec++; | |
1240 | if (first_p) | |
1241 | { | |
1242 | first_p = false; | |
1243 | ||
1244 | new_ds = ds; | |
1245 | } | |
1246 | else | |
1247 | new_ds = ds_merge (new_ds, ds); | |
1248 | } | |
1a83e602 | 1249 | else if (DEP_TYPE (dep) == REG_DEP_CONTROL) |
e2724e63 | 1250 | { |
1a83e602 BS |
1251 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED) |
1252 | { | |
1253 | n_control++; | |
1254 | modify_dep = dep; | |
1255 | } | |
1256 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
1257 | } | |
1258 | else if (DEP_REPLACE (dep) != NULL) | |
1259 | { | |
1260 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED) | |
1261 | { | |
1262 | n_replace++; | |
1263 | modify_dep = dep; | |
1264 | } | |
e2724e63 BS |
1265 | DEP_STATUS (dep) &= ~DEP_CANCELLED; |
1266 | } | |
1267 | } | |
1268 | ||
1a83e602 BS |
1269 | if (n_replace > 0 && n_control == 0 && n_spec == 0) |
1270 | { | |
1271 | if (!dbg_cnt (sched_breakdep)) | |
1272 | return HARD_DEP; | |
1273 | FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep) | |
1274 | { | |
1275 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
1276 | if (desc != NULL) | |
1277 | { | |
1278 | if (desc->insn == next && !for_backtrack) | |
1279 | { | |
1280 | gcc_assert (n_replace == 1); | |
1281 | apply_replacement (dep, true); | |
1282 | } | |
1283 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
1284 | } | |
1285 | } | |
1286 | return 0; | |
1287 | } | |
1288 | ||
1289 | else if (n_control == 1 && n_replace == 0 && n_spec == 0) | |
e2724e63 | 1290 | { |
f57aa6b0 DM |
1291 | rtx_insn *pro, *other; |
1292 | rtx new_pat; | |
e2724e63 BS |
1293 | rtx cond = NULL_RTX; |
1294 | bool success; | |
ce1ce33a | 1295 | rtx_insn *prev = NULL; |
e2724e63 BS |
1296 | int i; |
1297 | unsigned regno; | |
1298 | ||
1299 | if ((current_sched_info->flags & DO_PREDICATION) == 0 | |
1300 | || (ORIG_PAT (next) != NULL_RTX | |
1301 | && PREDICATED_PAT (next) == NULL_RTX)) | |
1302 | return HARD_DEP; | |
1303 | ||
1a83e602 | 1304 | pro = DEP_PRO (modify_dep); |
e2724e63 BS |
1305 | other = real_insn_for_shadow (pro); |
1306 | if (other != NULL_RTX) | |
1307 | pro = other; | |
1308 | ||
1309 | cond = sched_get_reverse_condition_uncached (pro); | |
1310 | regno = REGNO (XEXP (cond, 0)); | |
1311 | ||
1312 | /* Find the last scheduled insn that modifies the condition register. | |
89ff14c2 BS |
1313 | We can stop looking once we find the insn we depend on through the |
1314 | REG_DEP_CONTROL; if the condition register isn't modified after it, | |
1315 | we know that it still has the right value. */ | |
1316 | if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED) | |
9771b263 | 1317 | FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev) |
89ff14c2 BS |
1318 | { |
1319 | HARD_REG_SET t; | |
1320 | ||
356bf593 | 1321 | find_all_hard_reg_sets (prev, &t, true); |
89ff14c2 BS |
1322 | if (TEST_HARD_REG_BIT (t, regno)) |
1323 | return HARD_DEP; | |
1324 | if (prev == pro) | |
1325 | break; | |
1326 | } | |
e2724e63 BS |
1327 | if (ORIG_PAT (next) == NULL_RTX) |
1328 | { | |
1329 | ORIG_PAT (next) = PATTERN (next); | |
1330 | ||
1331 | new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next)); | |
1332 | success = haifa_change_pattern (next, new_pat); | |
1333 | if (!success) | |
1334 | return HARD_DEP; | |
1335 | PREDICATED_PAT (next) = new_pat; | |
1336 | } | |
1337 | else if (PATTERN (next) != PREDICATED_PAT (next)) | |
1338 | { | |
1339 | bool success = haifa_change_pattern (next, | |
1340 | PREDICATED_PAT (next)); | |
1341 | gcc_assert (success); | |
1342 | } | |
1a83e602 | 1343 | DEP_STATUS (modify_dep) |= DEP_CANCELLED; |
e2724e63 BS |
1344 | return DEP_CONTROL; |
1345 | } | |
1346 | ||
1347 | if (PREDICATED_PAT (next) != NULL_RTX) | |
1348 | { | |
1349 | int tick = INSN_TICK (next); | |
1350 | bool success = haifa_change_pattern (next, | |
1351 | ORIG_PAT (next)); | |
1352 | INSN_TICK (next) = tick; | |
1353 | gcc_assert (success); | |
1354 | } | |
1355 | ||
1356 | /* We can't handle the case where there are both speculative and control | |
1357 | dependencies, so we return HARD_DEP in such a case. Also fail if | |
1358 | we have speculative dependencies with not enough points, or more than | |
1359 | one control dependency. */ | |
1a83e602 | 1360 | if ((n_spec > 0 && (n_control > 0 || n_replace > 0)) |
e2724e63 BS |
1361 | || (n_spec > 0 |
1362 | /* Too few points? */ | |
1363 | && ds_weak (new_ds) < spec_info->data_weakness_cutoff) | |
1a83e602 BS |
1364 | || n_control > 0 |
1365 | || n_replace > 0) | |
e2724e63 BS |
1366 | return HARD_DEP; |
1367 | ||
1368 | return new_ds; | |
1369 | } | |
1370 | \f | |
9b69cf83 | 1371 | /* Pointer to the last instruction scheduled. */ |
ce1ce33a | 1372 | static rtx_insn *last_scheduled_insn; |
8c660648 | 1373 | |
5c24671b AO |
1374 | /* Pointer to the last nondebug instruction scheduled within the |
1375 | block, or the prev_head of the scheduling block. Used by | |
1376 | rank_for_schedule, so that insns independent of the last scheduled | |
1377 | insn will be preferred over dependent instructions. */ | |
90831096 | 1378 | static rtx_insn *last_nondebug_scheduled_insn; |
5c24671b | 1379 | |
9b69cf83 BS |
1380 | /* Pointer that iterates through the list of unscheduled insns if we |
1381 | have a dbg_cnt enabled. It always points at an insn prior to the | |
1382 | first unscheduled one. */ | |
ce1ce33a | 1383 | static rtx_insn *nonscheduled_insns_begin; |
9b69cf83 | 1384 | |
b198261f | 1385 | /* Compute cost of executing INSN. |
496d7bb0 MK |
1386 | This is the number of cycles between instruction issue and |
1387 | instruction results. */ | |
5fd8300b | 1388 | int |
ffc1ded5 | 1389 | insn_sched_cost (rtx_insn *insn) |
8c660648 | 1390 | { |
e855c69d AB |
1391 | int cost; |
1392 | ||
b16abbcb BC |
1393 | if (sched_fusion) |
1394 | return 0; | |
1395 | ||
e855c69d AB |
1396 | if (sel_sched_p ()) |
1397 | { | |
1398 | if (recog_memoized (insn) < 0) | |
1399 | return 0; | |
1400 | ||
1401 | cost = insn_default_latency (insn); | |
1402 | if (cost < 0) | |
1403 | cost = 0; | |
1404 | ||
1405 | return cost; | |
1406 | } | |
1407 | ||
1408 | cost = INSN_COST (insn); | |
8c660648 | 1409 | |
fae15c93 | 1410 | if (cost < 0) |
8c660648 | 1411 | { |
fae15c93 VM |
1412 | /* A USE insn, or something else we don't need to |
1413 | understand. We can't pass these directly to | |
1414 | result_ready_cost or insn_default_latency because it will | |
1415 | trigger a fatal error for unrecognizable insns. */ | |
1416 | if (recog_memoized (insn) < 0) | |
8c660648 | 1417 | { |
fae15c93 VM |
1418 | INSN_COST (insn) = 0; |
1419 | return 0; | |
8c660648 JL |
1420 | } |
1421 | else | |
1422 | { | |
fa0aee89 | 1423 | cost = insn_default_latency (insn); |
fae15c93 VM |
1424 | if (cost < 0) |
1425 | cost = 0; | |
1d088dee | 1426 | |
8c660648 JL |
1427 | INSN_COST (insn) = cost; |
1428 | } | |
1429 | } | |
1430 | ||
b198261f MK |
1431 | return cost; |
1432 | } | |
1433 | ||
1434 | /* Compute cost of dependence LINK. | |
1435 | This is the number of cycles between instruction issue and | |
078a70a1 AN |
1436 | instruction results. |
1437 | ??? We also use this function to call recog_memoized on all insns. */ | |
b198261f | 1438 | int |
e855c69d | 1439 | dep_cost_1 (dep_t link, dw_t dw) |
b198261f | 1440 | { |
9b517712 DM |
1441 | rtx_insn *insn = DEP_PRO (link); |
1442 | rtx_insn *used = DEP_CON (link); | |
b198261f | 1443 | int cost; |
8c660648 | 1444 | |
a19af9c4 BS |
1445 | if (DEP_COST (link) != UNKNOWN_DEP_COST) |
1446 | return DEP_COST (link); | |
1447 | ||
c203e8a7 | 1448 | if (delay_htab) |
26965010 BS |
1449 | { |
1450 | struct delay_pair *delay_entry; | |
1451 | delay_entry | |
c203e8a7 | 1452 | = delay_htab_i2->find_with_hash (used, htab_hash_pointer (used)); |
26965010 BS |
1453 | if (delay_entry) |
1454 | { | |
1455 | if (delay_entry->i1 == insn) | |
1456 | { | |
1457 | DEP_COST (link) = pair_delay (delay_entry); | |
1458 | return DEP_COST (link); | |
1459 | } | |
1460 | } | |
1461 | } | |
1462 | ||
fae15c93 VM |
1463 | /* A USE insn should never require the value used to be computed. |
1464 | This allows the computation of a function's result and parameter | |
ce18efcb | 1465 | values to overlap the return and call. We don't care about the |
dd5a833e | 1466 | dependence cost when only decreasing register pressure. */ |
fae15c93 | 1467 | if (recog_memoized (used) < 0) |
078a70a1 AN |
1468 | { |
1469 | cost = 0; | |
1470 | recog_memoized (insn); | |
1471 | } | |
fae15c93 | 1472 | else |
8c660648 | 1473 | { |
e2f6ff94 | 1474 | enum reg_note dep_type = DEP_TYPE (link); |
b198261f | 1475 | |
ffc1ded5 | 1476 | cost = insn_sched_cost (insn); |
496d7bb0 | 1477 | |
fa0aee89 | 1478 | if (INSN_CODE (insn) >= 0) |
197043f5 | 1479 | { |
496d7bb0 | 1480 | if (dep_type == REG_DEP_ANTI) |
fa0aee89 | 1481 | cost = 0; |
496d7bb0 | 1482 | else if (dep_type == REG_DEP_OUTPUT) |
fae15c93 | 1483 | { |
fa0aee89 PB |
1484 | cost = (insn_default_latency (insn) |
1485 | - insn_default_latency (used)); | |
1486 | if (cost <= 0) | |
1487 | cost = 1; | |
fae15c93 | 1488 | } |
fa0aee89 PB |
1489 | else if (bypass_p (insn)) |
1490 | cost = insn_latency (insn, used); | |
197043f5 | 1491 | } |
b8698a0f | 1492 | |
b8ec5764 | 1493 | |
b505225b TS |
1494 | if (targetm.sched.adjust_cost) |
1495 | cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost, | |
1496 | dw); | |
fae15c93 VM |
1497 | |
1498 | if (cost < 0) | |
1499 | cost = 0; | |
1500 | } | |
1d088dee | 1501 | |
a19af9c4 | 1502 | DEP_COST (link) = cost; |
8c660648 JL |
1503 | return cost; |
1504 | } | |
1505 | ||
e855c69d AB |
1506 | /* Compute cost of dependence LINK. |
1507 | This is the number of cycles between instruction issue and | |
1508 | instruction results. */ | |
1509 | int | |
1510 | dep_cost (dep_t link) | |
1511 | { | |
1512 | return dep_cost_1 (link, 0); | |
1513 | } | |
1514 | ||
1515 | /* Use this sel-sched.c friendly function in reorder2 instead of increasing | |
1516 | INSN_PRIORITY explicitly. */ | |
1517 | void | |
f57aa6b0 | 1518 | increase_insn_priority (rtx_insn *insn, int amount) |
e855c69d AB |
1519 | { |
1520 | if (!sel_sched_p ()) | |
1521 | { | |
1522 | /* We're dealing with haifa-sched.c INSN_PRIORITY. */ | |
1523 | if (INSN_PRIORITY_KNOWN (insn)) | |
1524 | INSN_PRIORITY (insn) += amount; | |
1525 | } | |
1526 | else | |
1527 | { | |
b8698a0f | 1528 | /* In sel-sched.c INSN_PRIORITY is not kept up to date. |
e855c69d AB |
1529 | Use EXPR_PRIORITY instead. */ |
1530 | sel_add_to_insn_priority (insn, amount); | |
1531 | } | |
1532 | } | |
1533 | ||
916fa4f0 MK |
1534 | /* Return 'true' if DEP should be included in priority calculations. */ |
1535 | static bool | |
1536 | contributes_to_priority_p (dep_t dep) | |
1537 | { | |
b5b8b0ac AO |
1538 | if (DEBUG_INSN_P (DEP_CON (dep)) |
1539 | || DEBUG_INSN_P (DEP_PRO (dep))) | |
1540 | return false; | |
1541 | ||
916fa4f0 MK |
1542 | /* Critical path is meaningful in block boundaries only. */ |
1543 | if (!current_sched_info->contributes_to_priority (DEP_CON (dep), | |
1544 | DEP_PRO (dep))) | |
1545 | return false; | |
1546 | ||
1a83e602 BS |
1547 | if (DEP_REPLACE (dep) != NULL) |
1548 | return false; | |
1549 | ||
916fa4f0 MK |
1550 | /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set, |
1551 | then speculative instructions will less likely be | |
1552 | scheduled. That is because the priority of | |
1553 | their producers will increase, and, thus, the | |
1554 | producers will more likely be scheduled, thus, | |
1555 | resolving the dependence. */ | |
e855c69d | 1556 | if (sched_deps_info->generate_spec_deps |
916fa4f0 MK |
1557 | && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH) |
1558 | && (DEP_STATUS (dep) & SPECULATIVE)) | |
1559 | return false; | |
8c660648 | 1560 | |
916fa4f0 MK |
1561 | return true; |
1562 | } | |
1563 | ||
7a1aca9c | 1564 | /* Compute the number of nondebug deps in list LIST for INSN. */ |
b5b8b0ac AO |
1565 | |
1566 | static int | |
90831096 | 1567 | dep_list_size (rtx_insn *insn, sd_list_types_def list) |
b5b8b0ac AO |
1568 | { |
1569 | sd_iterator_def sd_it; | |
1570 | dep_t dep; | |
1571 | int dbgcount = 0, nodbgcount = 0; | |
1572 | ||
1573 | if (!MAY_HAVE_DEBUG_INSNS) | |
7a1aca9c | 1574 | return sd_lists_size (insn, list); |
b5b8b0ac | 1575 | |
7a1aca9c | 1576 | FOR_EACH_DEP (insn, list, sd_it, dep) |
b5b8b0ac AO |
1577 | { |
1578 | if (DEBUG_INSN_P (DEP_CON (dep))) | |
1579 | dbgcount++; | |
f49b295a | 1580 | else if (!DEBUG_INSN_P (DEP_PRO (dep))) |
b5b8b0ac AO |
1581 | nodbgcount++; |
1582 | } | |
1583 | ||
7a1aca9c | 1584 | gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list)); |
b5b8b0ac AO |
1585 | |
1586 | return nodbgcount; | |
1587 | } | |
1588 | ||
b16abbcb BC |
1589 | bool sched_fusion; |
1590 | ||
916fa4f0 | 1591 | /* Compute the priority number for INSN. */ |
8c660648 | 1592 | static int |
079c81d0 | 1593 | priority (rtx_insn *insn, bool force_recompute) |
8c660648 | 1594 | { |
2c3c49de | 1595 | if (! INSN_P (insn)) |
8c660648 JL |
1596 | return 0; |
1597 | ||
c80b4100 | 1598 | /* We should not be interested in priority of an already scheduled insn. */ |
916fa4f0 MK |
1599 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED); |
1600 | ||
079c81d0 | 1601 | if (force_recompute || !INSN_PRIORITY_KNOWN (insn)) |
8c660648 | 1602 | { |
e855c69d | 1603 | int this_priority = -1; |
21e4c9a8 | 1604 | |
b16abbcb BC |
1605 | if (sched_fusion) |
1606 | { | |
1607 | int this_fusion_priority; | |
1608 | ||
1609 | targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY, | |
1610 | &this_fusion_priority, &this_priority); | |
1611 | INSN_FUSION_PRIORITY (insn) = this_fusion_priority; | |
1612 | } | |
1613 | else if (dep_list_size (insn, SD_LIST_FORW) == 0) | |
ffc1ded5 SB |
1614 | /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn |
1615 | has some forward deps but all of them are ignored by | |
b198261f MK |
1616 | contributes_to_priority hook. At the moment we set priority of |
1617 | such insn to 0. */ | |
ffc1ded5 | 1618 | this_priority = insn_sched_cost (insn); |
8c660648 | 1619 | else |
21e4c9a8 | 1620 | { |
dc01c3d1 | 1621 | rtx_insn *prev_first, *twin; |
496d7bb0 | 1622 | basic_block rec; |
8c660648 | 1623 | |
496d7bb0 MK |
1624 | /* For recovery check instructions we calculate priority slightly |
1625 | different than that of normal instructions. Instead of walking | |
b198261f MK |
1626 | through INSN_FORW_DEPS (check) list, we walk through |
1627 | INSN_FORW_DEPS list of each instruction in the corresponding | |
b8698a0f | 1628 | recovery block. */ |
8c660648 | 1629 | |
e855c69d AB |
1630 | /* Selective scheduling does not define RECOVERY_BLOCK macro. */ |
1631 | rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn); | |
fefa31b5 | 1632 | if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
1633 | { |
1634 | prev_first = PREV_INSN (insn); | |
1635 | twin = insn; | |
1636 | } | |
1637 | else | |
1638 | { | |
1639 | prev_first = NEXT_INSN (BB_HEAD (rec)); | |
1640 | twin = PREV_INSN (BB_END (rec)); | |
1641 | } | |
8c660648 | 1642 | |
496d7bb0 MK |
1643 | do |
1644 | { | |
e2f6ff94 MK |
1645 | sd_iterator_def sd_it; |
1646 | dep_t dep; | |
1647 | ||
1648 | FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep) | |
496d7bb0 | 1649 | { |
0e0ce50d | 1650 | rtx_insn *next; |
496d7bb0 | 1651 | int next_priority; |
b198261f MK |
1652 | |
1653 | next = DEP_CON (dep); | |
1654 | ||
496d7bb0 MK |
1655 | if (BLOCK_FOR_INSN (next) != rec) |
1656 | { | |
b198261f MK |
1657 | int cost; |
1658 | ||
916fa4f0 | 1659 | if (!contributes_to_priority_p (dep)) |
496d7bb0 | 1660 | continue; |
b198261f MK |
1661 | |
1662 | if (twin == insn) | |
1663 | cost = dep_cost (dep); | |
1664 | else | |
1665 | { | |
1666 | struct _dep _dep1, *dep1 = &_dep1; | |
1667 | ||
1668 | init_dep (dep1, insn, next, REG_DEP_ANTI); | |
1669 | ||
1670 | cost = dep_cost (dep1); | |
1671 | } | |
1672 | ||
1673 | next_priority = cost + priority (next); | |
496d7bb0 MK |
1674 | |
1675 | if (next_priority > this_priority) | |
1676 | this_priority = next_priority; | |
1677 | } | |
1678 | } | |
b8698a0f | 1679 | |
496d7bb0 | 1680 | twin = PREV_INSN (twin); |
21e4c9a8 | 1681 | } |
496d7bb0 | 1682 | while (twin != prev_first); |
21e4c9a8 | 1683 | } |
e855c69d AB |
1684 | |
1685 | if (this_priority < 0) | |
1686 | { | |
1687 | gcc_assert (this_priority == -1); | |
1688 | ||
ffc1ded5 | 1689 | this_priority = insn_sched_cost (insn); |
e855c69d AB |
1690 | } |
1691 | ||
8c660648 | 1692 | INSN_PRIORITY (insn) = this_priority; |
916fa4f0 | 1693 | INSN_PRIORITY_STATUS (insn) = 1; |
8c660648 | 1694 | } |
21e4c9a8 BS |
1695 | |
1696 | return INSN_PRIORITY (insn); | |
8c660648 JL |
1697 | } |
1698 | \f | |
8c660648 | 1699 | /* Macros and functions for keeping the priority queue sorted, and |
d91edf86 | 1700 | dealing with queuing and dequeuing of instructions. */ |
8c660648 | 1701 | |
7a1aca9c RS |
1702 | /* For each pressure class CL, set DEATH[CL] to the number of registers |
1703 | in that class that die in INSN. */ | |
1704 | ||
1705 | static void | |
f57aa6b0 | 1706 | calculate_reg_deaths (rtx_insn *insn, int *death) |
7a1aca9c RS |
1707 | { |
1708 | int i; | |
1709 | struct reg_use_data *use; | |
1710 | ||
1711 | for (i = 0; i < ira_pressure_classes_num; i++) | |
1712 | death[ira_pressure_classes[i]] = 0; | |
1713 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) | |
1714 | if (dying_use_p (use)) | |
1715 | mark_regno_birth_or_death (0, death, use->regno, true); | |
1716 | } | |
1717 | ||
ce18efcb VM |
1718 | /* Setup info about the current register pressure impact of scheduling |
1719 | INSN at the current scheduling point. */ | |
1720 | static void | |
f57aa6b0 | 1721 | setup_insn_reg_pressure_info (rtx_insn *insn) |
ce18efcb VM |
1722 | { |
1723 | int i, change, before, after, hard_regno; | |
1724 | int excess_cost_change; | |
ef4bddc2 | 1725 | machine_mode mode; |
ce18efcb VM |
1726 | enum reg_class cl; |
1727 | struct reg_pressure_data *pressure_info; | |
1728 | int *max_reg_pressure; | |
ce18efcb VM |
1729 | static int death[N_REG_CLASSES]; |
1730 | ||
c11b0b3b AO |
1731 | gcc_checking_assert (!DEBUG_INSN_P (insn)); |
1732 | ||
ce18efcb | 1733 | excess_cost_change = 0; |
7a1aca9c | 1734 | calculate_reg_deaths (insn, death); |
ce18efcb VM |
1735 | pressure_info = INSN_REG_PRESSURE (insn); |
1736 | max_reg_pressure = INSN_MAX_REG_PRESSURE (insn); | |
1737 | gcc_assert (pressure_info != NULL && max_reg_pressure != NULL); | |
1756cb66 | 1738 | for (i = 0; i < ira_pressure_classes_num; i++) |
ce18efcb | 1739 | { |
1756cb66 | 1740 | cl = ira_pressure_classes[i]; |
ce18efcb VM |
1741 | gcc_assert (curr_reg_pressure[cl] >= 0); |
1742 | change = (int) pressure_info[i].set_increase - death[cl]; | |
4dd9ac6c | 1743 | before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]); |
ce18efcb | 1744 | after = MAX (0, max_reg_pressure[i] + change |
4dd9ac6c | 1745 | - sched_class_regs_num[cl]); |
ce18efcb VM |
1746 | hard_regno = ira_class_hard_regs[cl][0]; |
1747 | gcc_assert (hard_regno >= 0); | |
1748 | mode = reg_raw_mode[hard_regno]; | |
1749 | excess_cost_change += ((after - before) | |
1750 | * (ira_memory_move_cost[mode][cl][0] | |
1751 | + ira_memory_move_cost[mode][cl][1])); | |
1752 | } | |
1753 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change; | |
1754 | } | |
7a1aca9c RS |
1755 | \f |
1756 | /* This is the first page of code related to SCHED_PRESSURE_MODEL. | |
1757 | It tries to make the scheduler take register pressure into account | |
1758 | without introducing too many unnecessary stalls. It hooks into the | |
1759 | main scheduling algorithm at several points: | |
1760 | ||
1761 | - Before scheduling starts, model_start_schedule constructs a | |
1762 | "model schedule" for the current block. This model schedule is | |
1763 | chosen solely to keep register pressure down. It does not take the | |
1764 | target's pipeline or the original instruction order into account, | |
1765 | except as a tie-breaker. It also doesn't work to a particular | |
1766 | pressure limit. | |
1767 | ||
1768 | This model schedule gives us an idea of what pressure can be | |
1769 | achieved for the block and gives us an example of a schedule that | |
1770 | keeps to that pressure. It also makes the final schedule less | |
1771 | dependent on the original instruction order. This is important | |
1772 | because the original order can either be "wide" (many values live | |
1773 | at once, such as in user-scheduled code) or "narrow" (few values | |
1774 | live at once, such as after loop unrolling, where several | |
1775 | iterations are executed sequentially). | |
1776 | ||
1777 | We do not apply this model schedule to the rtx stream. We simply | |
1778 | record it in model_schedule. We also compute the maximum pressure, | |
1779 | MP, that was seen during this schedule. | |
1780 | ||
1781 | - Instructions are added to the ready queue even if they require | |
1782 | a stall. The length of the stall is instead computed as: | |
1783 | ||
1784 | MAX (INSN_TICK (INSN) - clock_var, 0) | |
1785 | ||
1786 | (= insn_delay). This allows rank_for_schedule to choose between | |
1787 | introducing a deliberate stall or increasing pressure. | |
1788 | ||
1789 | - Before sorting the ready queue, model_set_excess_costs assigns | |
1790 | a pressure-based cost to each ready instruction in the queue. | |
1791 | This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE | |
1792 | (ECC for short) and is effectively measured in cycles. | |
1793 | ||
1794 | - rank_for_schedule ranks instructions based on: | |
1795 | ||
1796 | ECC (insn) + insn_delay (insn) | |
1797 | ||
1798 | then as: | |
1799 | ||
1800 | insn_delay (insn) | |
1801 | ||
1802 | So, for example, an instruction X1 with an ECC of 1 that can issue | |
1803 | now will win over an instruction X0 with an ECC of zero that would | |
1804 | introduce a stall of one cycle. However, an instruction X2 with an | |
1805 | ECC of 2 that can issue now will lose to both X0 and X1. | |
1806 | ||
1807 | - When an instruction is scheduled, model_recompute updates the model | |
1808 | schedule with the new pressures (some of which might now exceed the | |
1809 | original maximum pressure MP). model_update_limit_points then searches | |
1810 | for the new point of maximum pressure, if not already known. */ | |
1811 | ||
1812 | /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL | |
1813 | from surrounding debug information. */ | |
1814 | #define MODEL_BAR \ | |
1815 | ";;\t\t+------------------------------------------------------\n" | |
1816 | ||
1817 | /* Information about the pressure on a particular register class at a | |
1818 | particular point of the model schedule. */ | |
1819 | struct model_pressure_data { | |
1820 | /* The pressure at this point of the model schedule, or -1 if the | |
1821 | point is associated with an instruction that has already been | |
1822 | scheduled. */ | |
1823 | int ref_pressure; | |
1824 | ||
1825 | /* The maximum pressure during or after this point of the model schedule. */ | |
1826 | int max_pressure; | |
1827 | }; | |
1828 | ||
1829 | /* Per-instruction information that is used while building the model | |
1830 | schedule. Here, "schedule" refers to the model schedule rather | |
1831 | than the main schedule. */ | |
1832 | struct model_insn_info { | |
1833 | /* The instruction itself. */ | |
40a243d4 | 1834 | rtx_insn *insn; |
7a1aca9c RS |
1835 | |
1836 | /* If this instruction is in model_worklist, these fields link to the | |
1837 | previous (higher-priority) and next (lower-priority) instructions | |
1838 | in the list. */ | |
1839 | struct model_insn_info *prev; | |
1840 | struct model_insn_info *next; | |
1841 | ||
1842 | /* While constructing the schedule, QUEUE_INDEX describes whether an | |
1843 | instruction has already been added to the schedule (QUEUE_SCHEDULED), | |
1844 | is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE). | |
1845 | old_queue records the value that QUEUE_INDEX had before scheduling | |
1846 | started, so that we can restore it once the schedule is complete. */ | |
1847 | int old_queue; | |
1848 | ||
1849 | /* The relative importance of an unscheduled instruction. Higher | |
1850 | values indicate greater importance. */ | |
1851 | unsigned int model_priority; | |
1852 | ||
1853 | /* The length of the longest path of satisfied true dependencies | |
1854 | that leads to this instruction. */ | |
1855 | unsigned int depth; | |
1856 | ||
1857 | /* The length of the longest path of dependencies of any kind | |
1858 | that leads from this instruction. */ | |
1859 | unsigned int alap; | |
1860 | ||
1861 | /* The number of predecessor nodes that must still be scheduled. */ | |
1862 | int unscheduled_preds; | |
1863 | }; | |
1864 | ||
1865 | /* Information about the pressure limit for a particular register class. | |
1866 | This structure is used when applying a model schedule to the main | |
1867 | schedule. */ | |
1868 | struct model_pressure_limit { | |
1869 | /* The maximum register pressure seen in the original model schedule. */ | |
1870 | int orig_pressure; | |
1871 | ||
1872 | /* The maximum register pressure seen in the current model schedule | |
1873 | (which excludes instructions that have already been scheduled). */ | |
1874 | int pressure; | |
1875 | ||
1876 | /* The point of the current model schedule at which PRESSURE is first | |
1877 | reached. It is set to -1 if the value needs to be recomputed. */ | |
1878 | int point; | |
1879 | }; | |
1880 | ||
1881 | /* Describes a particular way of measuring register pressure. */ | |
1882 | struct model_pressure_group { | |
1883 | /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */ | |
1884 | struct model_pressure_limit limits[N_REG_CLASSES]; | |
1885 | ||
1886 | /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure | |
1887 | on register class ira_pressure_classes[PCI] at point POINT of the | |
1888 | current model schedule. A POINT of model_num_insns describes the | |
1889 | pressure at the end of the schedule. */ | |
1890 | struct model_pressure_data *model; | |
1891 | }; | |
1892 | ||
1893 | /* Index POINT gives the instruction at point POINT of the model schedule. | |
1894 | This array doesn't change during main scheduling. */ | |
f57aa6b0 | 1895 | static vec<rtx_insn *> model_schedule; |
7a1aca9c RS |
1896 | |
1897 | /* The list of instructions in the model worklist, sorted in order of | |
1898 | decreasing priority. */ | |
1899 | static struct model_insn_info *model_worklist; | |
1900 | ||
1901 | /* Index I describes the instruction with INSN_LUID I. */ | |
1902 | static struct model_insn_info *model_insns; | |
1903 | ||
1904 | /* The number of instructions in the model schedule. */ | |
1905 | static int model_num_insns; | |
1906 | ||
1907 | /* The index of the first instruction in model_schedule that hasn't yet been | |
1908 | added to the main schedule, or model_num_insns if all of them have. */ | |
1909 | static int model_curr_point; | |
1910 | ||
1911 | /* Describes the pressure before each instruction in the model schedule. */ | |
1912 | static struct model_pressure_group model_before_pressure; | |
1913 | ||
1914 | /* The first unused model_priority value (as used in model_insn_info). */ | |
1915 | static unsigned int model_next_priority; | |
1916 | ||
1917 | ||
1918 | /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP | |
1919 | at point POINT of the model schedule. */ | |
1920 | #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \ | |
1921 | (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)]) | |
1922 | ||
1923 | /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or | |
1924 | after point POINT of the model schedule. */ | |
1925 | #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \ | |
1926 | (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure) | |
1927 | ||
1928 | /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT | |
1929 | of the model schedule. */ | |
1930 | #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \ | |
1931 | (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure) | |
1932 | ||
1933 | /* Information about INSN that is used when creating the model schedule. */ | |
1934 | #define MODEL_INSN_INFO(INSN) \ | |
1935 | (&model_insns[INSN_LUID (INSN)]) | |
1936 | ||
1937 | /* The instruction at point POINT of the model schedule. */ | |
1938 | #define MODEL_INSN(POINT) \ | |
9771b263 | 1939 | (model_schedule[POINT]) |
7a1aca9c RS |
1940 | |
1941 | ||
1942 | /* Return INSN's index in the model schedule, or model_num_insns if it | |
1943 | doesn't belong to that schedule. */ | |
1944 | ||
1945 | static int | |
f57aa6b0 | 1946 | model_index (rtx_insn *insn) |
7a1aca9c RS |
1947 | { |
1948 | if (INSN_MODEL_INDEX (insn) == 0) | |
1949 | return model_num_insns; | |
1950 | return INSN_MODEL_INDEX (insn) - 1; | |
1951 | } | |
1952 | ||
1953 | /* Make sure that GROUP->limits is up-to-date for the current point | |
1954 | of the model schedule. */ | |
1955 | ||
1956 | static void | |
1957 | model_update_limit_points_in_group (struct model_pressure_group *group) | |
1958 | { | |
1959 | int pci, max_pressure, point; | |
1960 | ||
1961 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
1962 | { | |
1963 | /* We may have passed the final point at which the pressure in | |
1964 | group->limits[pci].pressure was reached. Update the limit if so. */ | |
1965 | max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci); | |
1966 | group->limits[pci].pressure = max_pressure; | |
1967 | ||
1968 | /* Find the point at which MAX_PRESSURE is first reached. We need | |
1969 | to search in three cases: | |
1970 | ||
1971 | - We've already moved past the previous pressure point. | |
1972 | In this case we search forward from model_curr_point. | |
1973 | ||
1974 | - We scheduled the previous point of maximum pressure ahead of | |
1975 | its position in the model schedule, but doing so didn't bring | |
1976 | the pressure point earlier. In this case we search forward | |
1977 | from that previous pressure point. | |
1978 | ||
1979 | - Scheduling an instruction early caused the maximum pressure | |
1980 | to decrease. In this case we will have set the pressure | |
1981 | point to -1, and we search forward from model_curr_point. */ | |
1982 | point = MAX (group->limits[pci].point, model_curr_point); | |
1983 | while (point < model_num_insns | |
1984 | && MODEL_REF_PRESSURE (group, point, pci) < max_pressure) | |
1985 | point++; | |
1986 | group->limits[pci].point = point; | |
1987 | ||
1988 | gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure); | |
1989 | gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure); | |
1990 | } | |
1991 | } | |
1992 | ||
1993 | /* Make sure that all register-pressure limits are up-to-date for the | |
1994 | current position in the model schedule. */ | |
1995 | ||
1996 | static void | |
1997 | model_update_limit_points (void) | |
1998 | { | |
1999 | model_update_limit_points_in_group (&model_before_pressure); | |
2000 | } | |
2001 | ||
2002 | /* Return the model_index of the last unscheduled use in chain USE | |
2003 | outside of USE's instruction. Return -1 if there are no other uses, | |
2004 | or model_num_insns if the register is live at the end of the block. */ | |
2005 | ||
2006 | static int | |
2007 | model_last_use_except (struct reg_use_data *use) | |
2008 | { | |
2009 | struct reg_use_data *next; | |
2010 | int last, index; | |
2011 | ||
2012 | last = -1; | |
2013 | for (next = use->next_regno_use; next != use; next = next->next_regno_use) | |
2014 | if (NONDEBUG_INSN_P (next->insn) | |
2015 | && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED) | |
2016 | { | |
2017 | index = model_index (next->insn); | |
2018 | if (index == model_num_insns) | |
2019 | return model_num_insns; | |
2020 | if (last < index) | |
2021 | last = index; | |
2022 | } | |
2023 | return last; | |
2024 | } | |
2025 | ||
2026 | /* An instruction with model_index POINT has just been scheduled, and it | |
2027 | adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1. | |
2028 | Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and | |
2029 | MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */ | |
2030 | ||
2031 | static void | |
2032 | model_start_update_pressure (struct model_pressure_group *group, | |
2033 | int point, int pci, int delta) | |
2034 | { | |
2035 | int next_max_pressure; | |
2036 | ||
2037 | if (point == model_num_insns) | |
2038 | { | |
2039 | /* The instruction wasn't part of the model schedule; it was moved | |
2040 | from a different block. Update the pressure for the end of | |
2041 | the model schedule. */ | |
2042 | MODEL_REF_PRESSURE (group, point, pci) += delta; | |
2043 | MODEL_MAX_PRESSURE (group, point, pci) += delta; | |
2044 | } | |
2045 | else | |
2046 | { | |
2047 | /* Record that this instruction has been scheduled. Nothing now | |
2048 | changes between POINT and POINT + 1, so get the maximum pressure | |
2049 | from the latter. If the maximum pressure decreases, the new | |
2050 | pressure point may be before POINT. */ | |
2051 | MODEL_REF_PRESSURE (group, point, pci) = -1; | |
2052 | next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci); | |
2053 | if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure) | |
2054 | { | |
2055 | MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure; | |
2056 | if (group->limits[pci].point == point) | |
2057 | group->limits[pci].point = -1; | |
2058 | } | |
2059 | } | |
2060 | } | |
2061 | ||
2062 | /* Record that scheduling a later instruction has changed the pressure | |
2063 | at point POINT of the model schedule by DELTA (which might be 0). | |
2064 | Update GROUP accordingly. Return nonzero if these changes might | |
2065 | trigger changes to previous points as well. */ | |
2066 | ||
2067 | static int | |
2068 | model_update_pressure (struct model_pressure_group *group, | |
2069 | int point, int pci, int delta) | |
2070 | { | |
2071 | int ref_pressure, max_pressure, next_max_pressure; | |
2072 | ||
2073 | /* If POINT hasn't yet been scheduled, update its pressure. */ | |
2074 | ref_pressure = MODEL_REF_PRESSURE (group, point, pci); | |
2075 | if (ref_pressure >= 0 && delta != 0) | |
2076 | { | |
2077 | ref_pressure += delta; | |
2078 | MODEL_REF_PRESSURE (group, point, pci) = ref_pressure; | |
2079 | ||
2080 | /* Check whether the maximum pressure in the overall schedule | |
2081 | has increased. (This means that the MODEL_MAX_PRESSURE of | |
4dd9ac6c | 2082 | every point <= POINT will need to increase too; see below.) */ |
7a1aca9c RS |
2083 | if (group->limits[pci].pressure < ref_pressure) |
2084 | group->limits[pci].pressure = ref_pressure; | |
2085 | ||
2086 | /* If we are at maximum pressure, and the maximum pressure | |
2087 | point was previously unknown or later than POINT, | |
2088 | bring it forward. */ | |
2089 | if (group->limits[pci].pressure == ref_pressure | |
2090 | && !IN_RANGE (group->limits[pci].point, 0, point)) | |
2091 | group->limits[pci].point = point; | |
2092 | ||
2093 | /* If POINT used to be the point of maximum pressure, but isn't | |
2094 | any longer, we need to recalculate it using a forward walk. */ | |
2095 | if (group->limits[pci].pressure > ref_pressure | |
2096 | && group->limits[pci].point == point) | |
2097 | group->limits[pci].point = -1; | |
2098 | } | |
2099 | ||
2100 | /* Update the maximum pressure at POINT. Changes here might also | |
2101 | affect the maximum pressure at POINT - 1. */ | |
2102 | next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci); | |
2103 | max_pressure = MAX (ref_pressure, next_max_pressure); | |
2104 | if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure) | |
2105 | { | |
2106 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
2107 | return 1; | |
2108 | } | |
2109 | return 0; | |
2110 | } | |
2111 | ||
2112 | /* INSN has just been scheduled. Update the model schedule accordingly. */ | |
2113 | ||
2114 | static void | |
f57aa6b0 | 2115 | model_recompute (rtx_insn *insn) |
7a1aca9c RS |
2116 | { |
2117 | struct { | |
2118 | int last_use; | |
2119 | int regno; | |
2120 | } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS]; | |
2121 | struct reg_use_data *use; | |
2122 | struct reg_pressure_data *reg_pressure; | |
2123 | int delta[N_REG_CLASSES]; | |
2124 | int pci, point, mix, new_last, cl, ref_pressure, queue; | |
2125 | unsigned int i, num_uses, num_pending_births; | |
2126 | bool print_p; | |
2127 | ||
2128 | /* The destinations of INSN were previously live from POINT onwards, but are | |
2129 | now live from model_curr_point onwards. Set up DELTA accordingly. */ | |
2130 | point = model_index (insn); | |
2131 | reg_pressure = INSN_REG_PRESSURE (insn); | |
2132 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2133 | { | |
2134 | cl = ira_pressure_classes[pci]; | |
2135 | delta[cl] = reg_pressure[pci].set_increase; | |
2136 | } | |
2137 | ||
2138 | /* Record which registers previously died at POINT, but which now die | |
2139 | before POINT. Adjust DELTA so that it represents the effect of | |
2140 | this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of | |
2141 | registers that will be born in the range [model_curr_point, POINT). */ | |
2142 | num_uses = 0; | |
2143 | num_pending_births = 0; | |
f380f608 | 2144 | bitmap_clear (tmp_bitmap); |
7a1aca9c RS |
2145 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) |
2146 | { | |
2147 | new_last = model_last_use_except (use); | |
f380f608 | 2148 | if (new_last < point && bitmap_set_bit (tmp_bitmap, use->regno)) |
7a1aca9c RS |
2149 | { |
2150 | gcc_assert (num_uses < ARRAY_SIZE (uses)); | |
2151 | uses[num_uses].last_use = new_last; | |
2152 | uses[num_uses].regno = use->regno; | |
2153 | /* This register is no longer live after POINT - 1. */ | |
2154 | mark_regno_birth_or_death (NULL, delta, use->regno, false); | |
2155 | num_uses++; | |
2156 | if (new_last >= 0) | |
2157 | num_pending_births++; | |
2158 | } | |
2159 | } | |
2160 | ||
2161 | /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT. | |
2162 | Also set each group pressure limit for POINT. */ | |
2163 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2164 | { | |
2165 | cl = ira_pressure_classes[pci]; | |
2166 | model_start_update_pressure (&model_before_pressure, | |
2167 | point, pci, delta[cl]); | |
2168 | } | |
2169 | ||
2170 | /* Walk the model schedule backwards, starting immediately before POINT. */ | |
2171 | print_p = false; | |
2172 | if (point != model_curr_point) | |
2173 | do | |
2174 | { | |
2175 | point--; | |
2176 | insn = MODEL_INSN (point); | |
2177 | queue = QUEUE_INDEX (insn); | |
2178 | ||
2179 | if (queue != QUEUE_SCHEDULED) | |
2180 | { | |
2181 | /* DELTA describes the effect of the move on the register pressure | |
2182 | after POINT. Make it describe the effect on the pressure | |
2183 | before POINT. */ | |
2184 | i = 0; | |
2185 | while (i < num_uses) | |
2186 | { | |
2187 | if (uses[i].last_use == point) | |
2188 | { | |
2189 | /* This register is now live again. */ | |
2190 | mark_regno_birth_or_death (NULL, delta, | |
2191 | uses[i].regno, true); | |
2192 | ||
2193 | /* Remove this use from the array. */ | |
2194 | uses[i] = uses[num_uses - 1]; | |
2195 | num_uses--; | |
2196 | num_pending_births--; | |
2197 | } | |
2198 | else | |
2199 | i++; | |
2200 | } | |
2201 | ||
2202 | if (sched_verbose >= 5) | |
2203 | { | |
7a1aca9c RS |
2204 | if (!print_p) |
2205 | { | |
2206 | fprintf (sched_dump, MODEL_BAR); | |
2207 | fprintf (sched_dump, ";;\t\t| New pressure for model" | |
2208 | " schedule\n"); | |
2209 | fprintf (sched_dump, MODEL_BAR); | |
2210 | print_p = true; | |
2211 | } | |
2212 | ||
7a1aca9c | 2213 | fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ", |
7eba871a SB |
2214 | point, INSN_UID (insn), |
2215 | str_pattern_slim (PATTERN (insn))); | |
7a1aca9c RS |
2216 | for (pci = 0; pci < ira_pressure_classes_num; pci++) |
2217 | { | |
2218 | cl = ira_pressure_classes[pci]; | |
2219 | ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure, | |
2220 | point, pci); | |
2221 | fprintf (sched_dump, " %s:[%d->%d]", | |
2222 | reg_class_names[ira_pressure_classes[pci]], | |
2223 | ref_pressure, ref_pressure + delta[cl]); | |
2224 | } | |
2225 | fprintf (sched_dump, "\n"); | |
2226 | } | |
2227 | } | |
2228 | ||
2229 | /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1 | |
2230 | might have changed as well. */ | |
2231 | mix = num_pending_births; | |
2232 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2233 | { | |
2234 | cl = ira_pressure_classes[pci]; | |
2235 | mix |= delta[cl]; | |
2236 | mix |= model_update_pressure (&model_before_pressure, | |
2237 | point, pci, delta[cl]); | |
2238 | } | |
2239 | } | |
2240 | while (mix && point > model_curr_point); | |
2241 | ||
2242 | if (print_p) | |
2243 | fprintf (sched_dump, MODEL_BAR); | |
2244 | } | |
1a83e602 BS |
2245 | |
2246 | /* After DEP, which was cancelled, has been resolved for insn NEXT, | |
2247 | check whether the insn's pattern needs restoring. */ | |
2248 | static bool | |
f57aa6b0 | 2249 | must_restore_pattern_p (rtx_insn *next, dep_t dep) |
1a83e602 BS |
2250 | { |
2251 | if (QUEUE_INDEX (next) == QUEUE_SCHEDULED) | |
2252 | return false; | |
2253 | ||
2254 | if (DEP_TYPE (dep) == REG_DEP_CONTROL) | |
2255 | { | |
2256 | gcc_assert (ORIG_PAT (next) != NULL_RTX); | |
2257 | gcc_assert (next == DEP_CON (dep)); | |
2258 | } | |
2259 | else | |
2260 | { | |
2261 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
2262 | if (desc->insn != next) | |
2263 | { | |
2264 | gcc_assert (*desc->loc == desc->orig); | |
2265 | return false; | |
2266 | } | |
2267 | } | |
2268 | return true; | |
2269 | } | |
7a1aca9c RS |
2270 | \f |
2271 | /* model_spill_cost (CL, P, P') returns the cost of increasing the | |
2272 | pressure on CL from P to P'. We use this to calculate a "base ECC", | |
2273 | baseECC (CL, X), for each pressure class CL and each instruction X. | |
2274 | Supposing X changes the pressure on CL from P to P', and that the | |
2275 | maximum pressure on CL in the current model schedule is MP', then: | |
2276 | ||
2277 | * if X occurs before or at the next point of maximum pressure in | |
2278 | the model schedule and P' > MP', then: | |
2279 | ||
2280 | baseECC (CL, X) = model_spill_cost (CL, MP, P') | |
2281 | ||
2282 | The idea is that the pressure after scheduling a fixed set of | |
2283 | instructions -- in this case, the set up to and including the | |
2284 | next maximum pressure point -- is going to be the same regardless | |
2285 | of the order; we simply want to keep the intermediate pressure | |
2286 | under control. Thus X has a cost of zero unless scheduling it | |
2287 | now would exceed MP'. | |
2288 | ||
2289 | If all increases in the set are by the same amount, no zero-cost | |
2290 | instruction will ever cause the pressure to exceed MP'. However, | |
2291 | if X is instead moved past an instruction X' with pressure in the | |
2292 | range (MP' - (P' - P), MP'), the pressure at X' will increase | |
2293 | beyond MP'. Since baseECC is very much a heuristic anyway, | |
2294 | it doesn't seem worth the overhead of tracking cases like these. | |
2295 | ||
2296 | The cost of exceeding MP' is always based on the original maximum | |
2297 | pressure MP. This is so that going 2 registers over the original | |
2298 | limit has the same cost regardless of whether it comes from two | |
2299 | separate +1 deltas or from a single +2 delta. | |
2300 | ||
2301 | * if X occurs after the next point of maximum pressure in the model | |
2302 | schedule and P' > P, then: | |
2303 | ||
2304 | baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P)) | |
2305 | ||
2306 | That is, if we move X forward across a point of maximum pressure, | |
2307 | and if X increases the pressure by P' - P, then we conservatively | |
2308 | assume that scheduling X next would increase the maximum pressure | |
2309 | by P' - P. Again, the cost of doing this is based on the original | |
2310 | maximum pressure MP, for the same reason as above. | |
2311 | ||
2312 | * if P' < P, P > MP, and X occurs at or after the next point of | |
2313 | maximum pressure, then: | |
2314 | ||
2315 | baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P) | |
ce18efcb | 2316 | |
7a1aca9c RS |
2317 | That is, if we have already exceeded the original maximum pressure MP, |
2318 | and if X might reduce the maximum pressure again -- or at least push | |
2319 | it further back, and thus allow more scheduling freedom -- it is given | |
2320 | a negative cost to reflect the improvement. | |
2321 | ||
2322 | * otherwise, | |
2323 | ||
2324 | baseECC (CL, X) = 0 | |
2325 | ||
2326 | In this case, X is not expected to affect the maximum pressure MP', | |
2327 | so it has zero cost. | |
2328 | ||
2329 | We then create a combined value baseECC (X) that is the sum of | |
2330 | baseECC (CL, X) for each pressure class CL. | |
2331 | ||
2332 | baseECC (X) could itself be used as the ECC value described above. | |
2333 | However, this is often too conservative, in the sense that it | |
2334 | tends to make high-priority instructions that increase pressure | |
2335 | wait too long in cases where introducing a spill would be better. | |
2336 | For this reason the final ECC is a priority-adjusted form of | |
2337 | baseECC (X). Specifically, we calculate: | |
2338 | ||
2339 | P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X) | |
2340 | baseP = MAX { P (X) | baseECC (X) <= 0 } | |
2341 | ||
2342 | Then: | |
2343 | ||
2344 | ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0) | |
2345 | ||
2346 | Thus an instruction's effect on pressure is ignored if it has a high | |
2347 | enough priority relative to the ones that don't increase pressure. | |
2348 | Negative values of baseECC (X) do not increase the priority of X | |
2349 | itself, but they do make it harder for other instructions to | |
2350 | increase the pressure further. | |
2351 | ||
2352 | This pressure cost is deliberately timid. The intention has been | |
2353 | to choose a heuristic that rarely interferes with the normal list | |
2354 | scheduler in cases where that scheduler would produce good code. | |
2355 | We simply want to curb some of its worst excesses. */ | |
2356 | ||
2357 | /* Return the cost of increasing the pressure in class CL from FROM to TO. | |
2358 | ||
2359 | Here we use the very simplistic cost model that every register above | |
4dd9ac6c | 2360 | sched_class_regs_num[CL] has a spill cost of 1. We could use other |
7a1aca9c RS |
2361 | measures instead, such as one based on MEMORY_MOVE_COST. However: |
2362 | ||
2363 | (1) In order for an instruction to be scheduled, the higher cost | |
2364 | would need to be justified in a single saving of that many stalls. | |
2365 | This is overly pessimistic, because the benefit of spilling is | |
2366 | often to avoid a sequence of several short stalls rather than | |
2367 | a single long one. | |
2368 | ||
2369 | (2) The cost is still arbitrary. Because we are not allocating | |
2370 | registers during scheduling, we have no way of knowing for | |
2371 | sure how many memory accesses will be required by each spill, | |
2372 | where the spills will be placed within the block, or even | |
2373 | which block(s) will contain the spills. | |
2374 | ||
2375 | So a higher cost than 1 is often too conservative in practice, | |
2376 | forcing blocks to contain unnecessary stalls instead of spill code. | |
2377 | The simple cost below seems to be the best compromise. It reduces | |
2378 | the interference with the normal list scheduler, which helps make | |
2379 | it more suitable for a default-on option. */ | |
2380 | ||
2381 | static int | |
2382 | model_spill_cost (int cl, int from, int to) | |
2383 | { | |
4dd9ac6c | 2384 | from = MAX (from, sched_class_regs_num[cl]); |
7a1aca9c RS |
2385 | return MAX (to, from) - from; |
2386 | } | |
2387 | ||
2388 | /* Return baseECC (ira_pressure_classes[PCI], POINT), given that | |
2389 | P = curr_reg_pressure[ira_pressure_classes[PCI]] and that | |
2390 | P' = P + DELTA. */ | |
2391 | ||
2392 | static int | |
2393 | model_excess_group_cost (struct model_pressure_group *group, | |
2394 | int point, int pci, int delta) | |
2395 | { | |
2396 | int pressure, cl; | |
2397 | ||
2398 | cl = ira_pressure_classes[pci]; | |
2399 | if (delta < 0 && point >= group->limits[pci].point) | |
2400 | { | |
2401 | pressure = MAX (group->limits[pci].orig_pressure, | |
2402 | curr_reg_pressure[cl] + delta); | |
2403 | return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]); | |
2404 | } | |
2405 | ||
2406 | if (delta > 0) | |
2407 | { | |
2408 | if (point > group->limits[pci].point) | |
2409 | pressure = group->limits[pci].pressure + delta; | |
2410 | else | |
2411 | pressure = curr_reg_pressure[cl] + delta; | |
2412 | ||
2413 | if (pressure > group->limits[pci].pressure) | |
2414 | return model_spill_cost (cl, group->limits[pci].orig_pressure, | |
2415 | pressure); | |
2416 | } | |
2417 | ||
2418 | return 0; | |
2419 | } | |
2420 | ||
2421 | /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump | |
2422 | if PRINT_P. */ | |
2423 | ||
2424 | static int | |
f57aa6b0 | 2425 | model_excess_cost (rtx_insn *insn, bool print_p) |
7a1aca9c RS |
2426 | { |
2427 | int point, pci, cl, cost, this_cost, delta; | |
2428 | struct reg_pressure_data *insn_reg_pressure; | |
2429 | int insn_death[N_REG_CLASSES]; | |
2430 | ||
2431 | calculate_reg_deaths (insn, insn_death); | |
2432 | point = model_index (insn); | |
2433 | insn_reg_pressure = INSN_REG_PRESSURE (insn); | |
2434 | cost = 0; | |
2435 | ||
2436 | if (print_p) | |
2437 | fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point, | |
2438 | INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn)); | |
2439 | ||
2440 | /* Sum up the individual costs for each register class. */ | |
2441 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2442 | { | |
2443 | cl = ira_pressure_classes[pci]; | |
2444 | delta = insn_reg_pressure[pci].set_increase - insn_death[cl]; | |
2445 | this_cost = model_excess_group_cost (&model_before_pressure, | |
2446 | point, pci, delta); | |
2447 | cost += this_cost; | |
2448 | if (print_p) | |
2449 | fprintf (sched_dump, " %s:[%d base cost %d]", | |
2450 | reg_class_names[cl], delta, this_cost); | |
2451 | } | |
2452 | ||
2453 | if (print_p) | |
2454 | fprintf (sched_dump, "\n"); | |
2455 | ||
2456 | return cost; | |
2457 | } | |
2458 | ||
2459 | /* Dump the next points of maximum pressure for GROUP. */ | |
2460 | ||
2461 | static void | |
2462 | model_dump_pressure_points (struct model_pressure_group *group) | |
2463 | { | |
2464 | int pci, cl; | |
2465 | ||
2466 | fprintf (sched_dump, ";;\t\t| pressure points"); | |
2467 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2468 | { | |
2469 | cl = ira_pressure_classes[pci]; | |
2470 | fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl], | |
2471 | curr_reg_pressure[cl], group->limits[pci].pressure); | |
2472 | if (group->limits[pci].point < model_num_insns) | |
2473 | fprintf (sched_dump, "%d:%d]", group->limits[pci].point, | |
2474 | INSN_UID (MODEL_INSN (group->limits[pci].point))); | |
2475 | else | |
2476 | fprintf (sched_dump, "end]"); | |
2477 | } | |
2478 | fprintf (sched_dump, "\n"); | |
2479 | } | |
2480 | ||
2481 | /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */ | |
2482 | ||
2483 | static void | |
ce1ce33a | 2484 | model_set_excess_costs (rtx_insn **insns, int count) |
7a1aca9c RS |
2485 | { |
2486 | int i, cost, priority_base, priority; | |
2487 | bool print_p; | |
2488 | ||
2489 | /* Record the baseECC value for each instruction in the model schedule, | |
4dd9ac6c | 2490 | except that negative costs are converted to zero ones now rather than |
7a1aca9c RS |
2491 | later. Do not assign a cost to debug instructions, since they must |
2492 | not change code-generation decisions. Experiments suggest we also | |
2493 | get better results by not assigning a cost to instructions from | |
2494 | a different block. | |
2495 | ||
2496 | Set PRIORITY_BASE to baseP in the block comment above. This is the | |
2497 | maximum priority of the "cheap" instructions, which should always | |
2498 | include the next model instruction. */ | |
2499 | priority_base = 0; | |
2500 | print_p = false; | |
2501 | for (i = 0; i < count; i++) | |
2502 | if (INSN_MODEL_INDEX (insns[i])) | |
2503 | { | |
2504 | if (sched_verbose >= 6 && !print_p) | |
2505 | { | |
2506 | fprintf (sched_dump, MODEL_BAR); | |
2507 | fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n"); | |
2508 | model_dump_pressure_points (&model_before_pressure); | |
2509 | fprintf (sched_dump, MODEL_BAR); | |
2510 | print_p = true; | |
2511 | } | |
2512 | cost = model_excess_cost (insns[i], print_p); | |
2513 | if (cost <= 0) | |
2514 | { | |
2515 | priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost; | |
2516 | priority_base = MAX (priority_base, priority); | |
2517 | cost = 0; | |
2518 | } | |
2519 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost; | |
2520 | } | |
2521 | if (print_p) | |
2522 | fprintf (sched_dump, MODEL_BAR); | |
2523 | ||
2524 | /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each | |
2525 | instruction. */ | |
2526 | for (i = 0; i < count; i++) | |
2527 | { | |
2528 | cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]); | |
2529 | priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]); | |
2530 | if (cost > 0 && priority > priority_base) | |
2531 | { | |
2532 | cost += priority_base - priority; | |
2533 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0); | |
2534 | } | |
2535 | } | |
2536 | } | |
2537 | \f | |
4bd12f3d MK |
2538 | |
2539 | /* Enum of rank_for_schedule heuristic decisions. */ | |
2540 | enum rfs_decision { | |
696d5fa1 | 2541 | RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2, |
4bd12f3d MK |
2542 | RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK, |
2543 | RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_SPECULATION, | |
2544 | RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX, | |
d8d9514c | 2545 | RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N }; |
4bd12f3d MK |
2546 | |
2547 | /* Corresponding strings for print outs. */ | |
2548 | static const char *rfs_str[RFS_N] = { | |
696d5fa1 | 2549 | "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2", |
4bd12f3d MK |
2550 | "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK", |
2551 | "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION", | |
2552 | "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX", | |
d8d9514c | 2553 | "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" }; |
4bd12f3d MK |
2554 | |
2555 | /* Statistical breakdown of rank_for_schedule decisions. */ | |
50686850 | 2556 | struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; }; |
4bd12f3d MK |
2557 | static rank_for_schedule_stats_t rank_for_schedule_stats; |
2558 | ||
b75ae7f4 MK |
2559 | /* Return the result of comparing insns TMP and TMP2 and update |
2560 | Rank_For_Schedule statistics. */ | |
4bd12f3d | 2561 | static int |
b75ae7f4 | 2562 | rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2) |
4bd12f3d MK |
2563 | { |
2564 | ++rank_for_schedule_stats.stats[decision]; | |
b75ae7f4 MK |
2565 | if (result < 0) |
2566 | INSN_LAST_RFS_WIN (tmp) = decision; | |
2567 | else if (result > 0) | |
2568 | INSN_LAST_RFS_WIN (tmp2) = decision; | |
2569 | else | |
2570 | gcc_unreachable (); | |
4bd12f3d MK |
2571 | return result; |
2572 | } | |
2573 | ||
63387a85 MK |
2574 | /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while |
2575 | keeping normal insns in original order. */ | |
2576 | ||
2577 | static int | |
2578 | rank_for_schedule_debug (const void *x, const void *y) | |
2579 | { | |
2580 | rtx_insn *tmp = *(rtx_insn * const *) y; | |
2581 | rtx_insn *tmp2 = *(rtx_insn * const *) x; | |
2582 | ||
2583 | /* Schedule debug insns as early as possible. */ | |
2584 | if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2)) | |
696d5fa1 | 2585 | return -1; |
63387a85 | 2586 | else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2)) |
696d5fa1 | 2587 | return 1; |
63387a85 | 2588 | else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2)) |
696d5fa1 | 2589 | return INSN_LUID (tmp) - INSN_LUID (tmp2); |
63387a85 MK |
2590 | else |
2591 | return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp); | |
2592 | } | |
2593 | ||
8c660648 JL |
2594 | /* Returns a positive value if x is preferred; returns a negative value if |
2595 | y is preferred. Should never return 0, since that will make the sort | |
2596 | unstable. */ | |
2597 | ||
2598 | static int | |
1d088dee | 2599 | rank_for_schedule (const void *x, const void *y) |
8c660648 | 2600 | { |
ce1ce33a DM |
2601 | rtx_insn *tmp = *(rtx_insn * const *) y; |
2602 | rtx_insn *tmp2 = *(rtx_insn * const *) x; | |
b198261f | 2603 | int tmp_class, tmp2_class; |
f20f2613 | 2604 | int val, priority_val, info_val, diff; |
8c660648 | 2605 | |
f20f2613 VM |
2606 | if (live_range_shrinkage_p) |
2607 | { | |
2608 | /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse | |
2609 | code. */ | |
2610 | gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED); | |
2611 | if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0 | |
2612 | || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0) | |
2613 | && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) | |
2614 | - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0) | |
b75ae7f4 | 2615 | return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2); |
f20f2613 VM |
2616 | /* Sort by INSN_LUID (original insn order), so that we make the |
2617 | sort stable. This minimizes instruction movement, thus | |
2618 | minimizing sched's effect on debugging and cross-jumping. */ | |
4bd12f3d | 2619 | return rfs_result (RFS_LIVE_RANGE_SHRINK2, |
b75ae7f4 | 2620 | INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); |
f20f2613 VM |
2621 | } |
2622 | ||
58fb7809 | 2623 | /* The insn in a schedule group should be issued the first. */ |
b8698a0f | 2624 | if (flag_sched_group_heuristic && |
ee4764a8 | 2625 | SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2)) |
b75ae7f4 MK |
2626 | return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1, |
2627 | tmp, tmp2); | |
58fb7809 | 2628 | |
916fa4f0 MK |
2629 | /* Make sure that priority of TMP and TMP2 are initialized. */ |
2630 | gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2)); | |
2631 | ||
b16abbcb BC |
2632 | if (sched_fusion) |
2633 | { | |
2634 | /* The instruction that has the same fusion priority as the last | |
2635 | instruction is the instruction we picked next. If that is not | |
2636 | the case, we sort ready list firstly by fusion priority, then | |
2637 | by priority, and at last by INSN_LUID. */ | |
2638 | int a = INSN_FUSION_PRIORITY (tmp); | |
2639 | int b = INSN_FUSION_PRIORITY (tmp2); | |
2640 | int last = -1; | |
2641 | ||
2642 | if (last_nondebug_scheduled_insn | |
2643 | && !NOTE_P (last_nondebug_scheduled_insn) | |
2644 | && BLOCK_FOR_INSN (tmp) | |
2645 | == BLOCK_FOR_INSN (last_nondebug_scheduled_insn)) | |
2646 | last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn); | |
2647 | ||
2648 | if (a != last && b != last) | |
2649 | { | |
2650 | if (a == b) | |
2651 | { | |
2652 | a = INSN_PRIORITY (tmp); | |
2653 | b = INSN_PRIORITY (tmp2); | |
2654 | } | |
2655 | if (a != b) | |
2656 | return rfs_result (RFS_FUSION, b - a, tmp, tmp2); | |
2657 | else | |
2658 | return rfs_result (RFS_FUSION, | |
2659 | INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); | |
2660 | } | |
2661 | else if (a == b) | |
2662 | { | |
2663 | gcc_assert (last_nondebug_scheduled_insn | |
2664 | && !NOTE_P (last_nondebug_scheduled_insn)); | |
2665 | last = INSN_PRIORITY (last_nondebug_scheduled_insn); | |
2666 | ||
2667 | a = abs (INSN_PRIORITY (tmp) - last); | |
2668 | b = abs (INSN_PRIORITY (tmp2) - last); | |
2669 | if (a != b) | |
2670 | return rfs_result (RFS_FUSION, a - b, tmp, tmp2); | |
2671 | else | |
2672 | return rfs_result (RFS_FUSION, | |
2673 | INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); | |
2674 | } | |
2675 | else if (a == last) | |
2676 | return rfs_result (RFS_FUSION, -1, tmp, tmp2); | |
2677 | else | |
2678 | return rfs_result (RFS_FUSION, 1, tmp, tmp2); | |
2679 | } | |
2680 | ||
7a1aca9c | 2681 | if (sched_pressure != SCHED_PRESSURE_NONE) |
ce18efcb | 2682 | { |
ce18efcb VM |
2683 | /* Prefer insn whose scheduling results in the smallest register |
2684 | pressure excess. */ | |
2685 | if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) | |
7a1aca9c | 2686 | + insn_delay (tmp) |
ce18efcb | 2687 | - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) |
7a1aca9c | 2688 | - insn_delay (tmp2)))) |
b75ae7f4 | 2689 | return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2); |
ce18efcb VM |
2690 | } |
2691 | ||
7a1aca9c | 2692 | if (sched_pressure != SCHED_PRESSURE_NONE |
88366b18 MK |
2693 | && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var) |
2694 | && INSN_TICK (tmp2) != INSN_TICK (tmp)) | |
ce18efcb | 2695 | { |
88366b18 | 2696 | diff = INSN_TICK (tmp) - INSN_TICK (tmp2); |
b75ae7f4 | 2697 | return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2); |
ce18efcb | 2698 | } |
26965010 BS |
2699 | |
2700 | /* If we are doing backtracking in this schedule, prefer insns that | |
2701 | have forward dependencies with negative cost against an insn that | |
2702 | was already scheduled. */ | |
2703 | if (current_sched_info->flags & DO_BACKTRACKING) | |
2704 | { | |
2705 | priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp); | |
2706 | if (priority_val) | |
b75ae7f4 | 2707 | return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2); |
26965010 BS |
2708 | } |
2709 | ||
63de6c74 | 2710 | /* Prefer insn with higher priority. */ |
8c660648 | 2711 | priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp); |
30028c85 | 2712 | |
ee4764a8 | 2713 | if (flag_sched_critical_path_heuristic && priority_val) |
b75ae7f4 | 2714 | return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2); |
b8698a0f | 2715 | |
340c7904 MK |
2716 | if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) >= 0) |
2717 | { | |
2718 | int autopref = autopref_rank_for_schedule (tmp, tmp2); | |
2719 | if (autopref != 0) | |
2720 | return autopref; | |
2721 | } | |
2722 | ||
496d7bb0 | 2723 | /* Prefer speculative insn with greater dependencies weakness. */ |
ee4764a8 | 2724 | if (flag_sched_spec_insn_heuristic && spec_info) |
496d7bb0 MK |
2725 | { |
2726 | ds_t ds1, ds2; | |
2727 | dw_t dw1, dw2; | |
2728 | int dw; | |
2729 | ||
2730 | ds1 = TODO_SPEC (tmp) & SPECULATIVE; | |
2731 | if (ds1) | |
e855c69d | 2732 | dw1 = ds_weak (ds1); |
496d7bb0 MK |
2733 | else |
2734 | dw1 = NO_DEP_WEAK; | |
b8698a0f | 2735 | |
496d7bb0 MK |
2736 | ds2 = TODO_SPEC (tmp2) & SPECULATIVE; |
2737 | if (ds2) | |
e855c69d | 2738 | dw2 = ds_weak (ds2); |
496d7bb0 MK |
2739 | else |
2740 | dw2 = NO_DEP_WEAK; | |
2741 | ||
2742 | dw = dw2 - dw1; | |
2743 | if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8)) | |
b75ae7f4 | 2744 | return rfs_result (RFS_SPECULATION, dw, tmp, tmp2); |
496d7bb0 MK |
2745 | } |
2746 | ||
1708fd40 | 2747 | info_val = (*current_sched_info->rank) (tmp, tmp2); |
c3284718 | 2748 | if (flag_sched_rank_heuristic && info_val) |
b75ae7f4 | 2749 | return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2); |
8c660648 | 2750 | |
b5b8b0ac AO |
2751 | /* Compare insns based on their relation to the last scheduled |
2752 | non-debug insn. */ | |
5c24671b | 2753 | if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn) |
8c660648 | 2754 | { |
e2f6ff94 MK |
2755 | dep_t dep1; |
2756 | dep_t dep2; | |
90831096 | 2757 | rtx_insn *last = last_nondebug_scheduled_insn; |
e2f6ff94 | 2758 | |
8c660648 JL |
2759 | /* Classify the instructions into three classes: |
2760 | 1) Data dependent on last schedule insn. | |
2761 | 2) Anti/Output dependent on last scheduled insn. | |
2762 | 3) Independent of last scheduled insn, or has latency of one. | |
2763 | Choose the insn from the highest numbered class if different. */ | |
b5b8b0ac | 2764 | dep1 = sd_find_dep_between (last, tmp, true); |
b198261f | 2765 | |
e2f6ff94 | 2766 | if (dep1 == NULL || dep_cost (dep1) == 1) |
8c660648 | 2767 | tmp_class = 3; |
b198261f | 2768 | else if (/* Data dependence. */ |
e2f6ff94 | 2769 | DEP_TYPE (dep1) == REG_DEP_TRUE) |
8c660648 JL |
2770 | tmp_class = 1; |
2771 | else | |
2772 | tmp_class = 2; | |
2773 | ||
b5b8b0ac | 2774 | dep2 = sd_find_dep_between (last, tmp2, true); |
b198261f | 2775 | |
e2f6ff94 | 2776 | if (dep2 == NULL || dep_cost (dep2) == 1) |
8c660648 | 2777 | tmp2_class = 3; |
b198261f | 2778 | else if (/* Data dependence. */ |
e2f6ff94 | 2779 | DEP_TYPE (dep2) == REG_DEP_TRUE) |
8c660648 JL |
2780 | tmp2_class = 1; |
2781 | else | |
2782 | tmp2_class = 2; | |
2783 | ||
2784 | if ((val = tmp2_class - tmp_class)) | |
b75ae7f4 | 2785 | return rfs_result (RFS_LAST_INSN, val, tmp, tmp2); |
8c660648 JL |
2786 | } |
2787 | ||
7a1aca9c | 2788 | /* Prefer instructions that occur earlier in the model schedule. */ |
18fbe394 | 2789 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
7a1aca9c | 2790 | { |
7a1aca9c | 2791 | diff = model_index (tmp) - model_index (tmp2); |
18fbe394 WD |
2792 | if (diff != 0) |
2793 | return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2); | |
7a1aca9c RS |
2794 | } |
2795 | ||
7a403706 | 2796 | /* Prefer the insn which has more later insns that depend on it. |
2db45993 JL |
2797 | This gives the scheduler more freedom when scheduling later |
2798 | instructions at the expense of added register pressure. */ | |
2db45993 | 2799 | |
7a1aca9c RS |
2800 | val = (dep_list_size (tmp2, SD_LIST_FORW) |
2801 | - dep_list_size (tmp, SD_LIST_FORW)); | |
2db45993 | 2802 | |
ee4764a8 | 2803 | if (flag_sched_dep_count_heuristic && val != 0) |
b75ae7f4 | 2804 | return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2); |
7a403706 | 2805 | |
d8d9514c VL |
2806 | /* Sort by INSN_COST rather than INSN_LUID. This means that instructions |
2807 | which take longer to execute are prioritised and it leads to more | |
2808 | dual-issue opportunities on in-order cores which have this feature. */ | |
2809 | ||
2810 | if (INSN_COST (tmp) != INSN_COST (tmp2)) | |
2811 | return rfs_result (RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp), | |
2812 | tmp, tmp2); | |
2813 | ||
8c660648 JL |
2814 | /* If insns are equally good, sort by INSN_LUID (original insn order), |
2815 | so that we make the sort stable. This minimizes instruction movement, | |
2816 | thus minimizing sched's effect on debugging and cross-jumping. */ | |
b75ae7f4 | 2817 | return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); |
8c660648 JL |
2818 | } |
2819 | ||
2820 | /* Resort the array A in which only element at index N may be out of order. */ | |
2821 | ||
cbb13457 | 2822 | HAIFA_INLINE static void |
ce1ce33a | 2823 | swap_sort (rtx_insn **a, int n) |
8c660648 | 2824 | { |
ce1ce33a | 2825 | rtx_insn *insn = a[n - 1]; |
8c660648 JL |
2826 | int i = n - 2; |
2827 | ||
2828 | while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0) | |
2829 | { | |
2830 | a[i + 1] = a[i]; | |
2831 | i -= 1; | |
2832 | } | |
2833 | a[i + 1] = insn; | |
2834 | } | |
2835 | ||
8c660648 JL |
2836 | /* Add INSN to the insn queue so that it can be executed at least |
2837 | N_CYCLES after the currently executing insn. Preserve insns | |
ec4efea9 BS |
2838 | chain for debugging purposes. REASON will be printed in debugging |
2839 | output. */ | |
8c660648 | 2840 | |
cbb13457 | 2841 | HAIFA_INLINE static void |
ce1ce33a | 2842 | queue_insn (rtx_insn *insn, int n_cycles, const char *reason) |
8c660648 JL |
2843 | { |
2844 | int next_q = NEXT_Q_AFTER (q_ptr, n_cycles); | |
3dc99c19 | 2845 | rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]); |
26965010 | 2846 | int new_tick; |
63f54b1a MK |
2847 | |
2848 | gcc_assert (n_cycles <= max_insn_queue_index); | |
b5b8b0ac | 2849 | gcc_assert (!DEBUG_INSN_P (insn)); |
63f54b1a | 2850 | |
8c660648 JL |
2851 | insn_queue[next_q] = link; |
2852 | q_size += 1; | |
2853 | ||
2854 | if (sched_verbose >= 2) | |
2855 | { | |
1708fd40 BS |
2856 | fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ", |
2857 | (*current_sched_info->print_insn) (insn, 0)); | |
8c660648 | 2858 | |
ec4efea9 | 2859 | fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason); |
8c660648 | 2860 | } |
e855c69d | 2861 | |
63f54b1a | 2862 | QUEUE_INDEX (insn) = next_q; |
26965010 BS |
2863 | |
2864 | if (current_sched_info->flags & DO_BACKTRACKING) | |
2865 | { | |
2866 | new_tick = clock_var + n_cycles; | |
2867 | if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick) | |
2868 | INSN_TICK (insn) = new_tick; | |
2869 | ||
2870 | if (INSN_EXACT_TICK (insn) != INVALID_TICK | |
2871 | && INSN_EXACT_TICK (insn) < clock_var + n_cycles) | |
2872 | { | |
2873 | must_backtrack = true; | |
2874 | if (sched_verbose >= 2) | |
2875 | fprintf (sched_dump, ";;\t\tcausing a backtrack.\n"); | |
2876 | } | |
2877 | } | |
63f54b1a MK |
2878 | } |
2879 | ||
2880 | /* Remove INSN from queue. */ | |
2881 | static void | |
f57aa6b0 | 2882 | queue_remove (rtx_insn *insn) |
63f54b1a MK |
2883 | { |
2884 | gcc_assert (QUEUE_INDEX (insn) >= 0); | |
2885 | remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]); | |
2886 | q_size--; | |
2887 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; | |
176f9a7b BS |
2888 | } |
2889 | ||
2890 | /* Return a pointer to the bottom of the ready list, i.e. the insn | |
2891 | with the lowest priority. */ | |
2892 | ||
ce1ce33a | 2893 | rtx_insn ** |
1d088dee | 2894 | ready_lastpos (struct ready_list *ready) |
176f9a7b | 2895 | { |
63f54b1a | 2896 | gcc_assert (ready->n_ready >= 1); |
176f9a7b BS |
2897 | return ready->vec + ready->first - ready->n_ready + 1; |
2898 | } | |
2899 | ||
63f54b1a | 2900 | /* Add an element INSN to the ready list so that it ends up with the |
917f1b7e | 2901 | lowest/highest priority depending on FIRST_P. */ |
176f9a7b | 2902 | |
63f54b1a | 2903 | HAIFA_INLINE static void |
ce1ce33a | 2904 | ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p) |
176f9a7b | 2905 | { |
63f54b1a | 2906 | if (!first_p) |
176f9a7b | 2907 | { |
63f54b1a MK |
2908 | if (ready->first == ready->n_ready) |
2909 | { | |
2910 | memmove (ready->vec + ready->veclen - ready->n_ready, | |
2911 | ready_lastpos (ready), | |
2912 | ready->n_ready * sizeof (rtx)); | |
2913 | ready->first = ready->veclen - 1; | |
2914 | } | |
2915 | ready->vec[ready->first - ready->n_ready] = insn; | |
176f9a7b | 2916 | } |
63f54b1a MK |
2917 | else |
2918 | { | |
2919 | if (ready->first == ready->veclen - 1) | |
2920 | { | |
2921 | if (ready->n_ready) | |
2922 | /* ready_lastpos() fails when called with (ready->n_ready == 0). */ | |
2923 | memmove (ready->vec + ready->veclen - ready->n_ready - 1, | |
2924 | ready_lastpos (ready), | |
2925 | ready->n_ready * sizeof (rtx)); | |
2926 | ready->first = ready->veclen - 2; | |
2927 | } | |
2928 | ready->vec[++(ready->first)] = insn; | |
2929 | } | |
2930 | ||
176f9a7b | 2931 | ready->n_ready++; |
b5b8b0ac AO |
2932 | if (DEBUG_INSN_P (insn)) |
2933 | ready->n_debug++; | |
63f54b1a MK |
2934 | |
2935 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY); | |
2936 | QUEUE_INDEX (insn) = QUEUE_READY; | |
26965010 BS |
2937 | |
2938 | if (INSN_EXACT_TICK (insn) != INVALID_TICK | |
2939 | && INSN_EXACT_TICK (insn) < clock_var) | |
2940 | { | |
2941 | must_backtrack = true; | |
2942 | } | |
176f9a7b | 2943 | } |
8c660648 | 2944 | |
176f9a7b BS |
2945 | /* Remove the element with the highest priority from the ready list and |
2946 | return it. */ | |
2947 | ||
ce1ce33a | 2948 | HAIFA_INLINE static rtx_insn * |
1d088dee | 2949 | ready_remove_first (struct ready_list *ready) |
176f9a7b | 2950 | { |
ce1ce33a | 2951 | rtx_insn *t; |
b8698a0f | 2952 | |
535a42b1 | 2953 | gcc_assert (ready->n_ready); |
176f9a7b BS |
2954 | t = ready->vec[ready->first--]; |
2955 | ready->n_ready--; | |
b5b8b0ac AO |
2956 | if (DEBUG_INSN_P (t)) |
2957 | ready->n_debug--; | |
176f9a7b BS |
2958 | /* If the queue becomes empty, reset it. */ |
2959 | if (ready->n_ready == 0) | |
2960 | ready->first = ready->veclen - 1; | |
63f54b1a MK |
2961 | |
2962 | gcc_assert (QUEUE_INDEX (t) == QUEUE_READY); | |
2963 | QUEUE_INDEX (t) = QUEUE_NOWHERE; | |
2964 | ||
176f9a7b BS |
2965 | return t; |
2966 | } | |
2967 | ||
fae15c93 VM |
2968 | /* The following code implements multi-pass scheduling for the first |
2969 | cycle. In other words, we will try to choose ready insn which | |
2970 | permits to start maximum number of insns on the same cycle. */ | |
2971 | ||
2972 | /* Return a pointer to the element INDEX from the ready. INDEX for | |
2973 | insn with the highest priority is 0, and the lowest priority has | |
2974 | N_READY - 1. */ | |
2975 | ||
ce1ce33a | 2976 | rtx_insn * |
1d088dee | 2977 | ready_element (struct ready_list *ready, int index) |
fae15c93 | 2978 | { |
535a42b1 | 2979 | gcc_assert (ready->n_ready && index < ready->n_ready); |
b8698a0f | 2980 | |
fae15c93 VM |
2981 | return ready->vec[ready->first - index]; |
2982 | } | |
2983 | ||
2984 | /* Remove the element INDEX from the ready list and return it. INDEX | |
2985 | for insn with the highest priority is 0, and the lowest priority | |
2986 | has N_READY - 1. */ | |
2987 | ||
ce1ce33a | 2988 | HAIFA_INLINE static rtx_insn * |
1d088dee | 2989 | ready_remove (struct ready_list *ready, int index) |
fae15c93 | 2990 | { |
ce1ce33a | 2991 | rtx_insn *t; |
fae15c93 VM |
2992 | int i; |
2993 | ||
2994 | if (index == 0) | |
2995 | return ready_remove_first (ready); | |
535a42b1 | 2996 | gcc_assert (ready->n_ready && index < ready->n_ready); |
fae15c93 VM |
2997 | t = ready->vec[ready->first - index]; |
2998 | ready->n_ready--; | |
b5b8b0ac AO |
2999 | if (DEBUG_INSN_P (t)) |
3000 | ready->n_debug--; | |
fae15c93 VM |
3001 | for (i = index; i < ready->n_ready; i++) |
3002 | ready->vec[ready->first - i] = ready->vec[ready->first - i - 1]; | |
63f54b1a | 3003 | QUEUE_INDEX (t) = QUEUE_NOWHERE; |
fae15c93 VM |
3004 | return t; |
3005 | } | |
3006 | ||
63f54b1a MK |
3007 | /* Remove INSN from the ready list. */ |
3008 | static void | |
90831096 | 3009 | ready_remove_insn (rtx_insn *insn) |
63f54b1a MK |
3010 | { |
3011 | int i; | |
3012 | ||
3013 | for (i = 0; i < readyp->n_ready; i++) | |
3014 | if (ready_element (readyp, i) == insn) | |
3015 | { | |
3016 | ready_remove (readyp, i); | |
3017 | return; | |
3018 | } | |
3019 | gcc_unreachable (); | |
3020 | } | |
fae15c93 | 3021 | |
4bd12f3d MK |
3022 | /* Calculate difference of two statistics set WAS and NOW. |
3023 | Result returned in WAS. */ | |
3024 | static void | |
3025 | rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was, | |
3026 | const rank_for_schedule_stats_t *now) | |
3027 | { | |
3028 | for (int i = 0; i < RFS_N; ++i) | |
3029 | was->stats[i] = now->stats[i] - was->stats[i]; | |
3030 | } | |
3031 | ||
3032 | /* Print rank_for_schedule statistics. */ | |
3033 | static void | |
3034 | print_rank_for_schedule_stats (const char *prefix, | |
b75ae7f4 MK |
3035 | const rank_for_schedule_stats_t *stats, |
3036 | struct ready_list *ready) | |
4bd12f3d MK |
3037 | { |
3038 | for (int i = 0; i < RFS_N; ++i) | |
3039 | if (stats->stats[i]) | |
b75ae7f4 MK |
3040 | { |
3041 | fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]); | |
3042 | ||
3043 | if (ready != NULL) | |
3044 | /* Print out insns that won due to RFS_<I>. */ | |
3045 | { | |
3046 | rtx_insn **p = ready_lastpos (ready); | |
3047 | ||
3048 | fprintf (sched_dump, ":"); | |
3049 | /* Start with 1 since least-priority insn didn't have any wins. */ | |
3050 | for (int j = 1; j < ready->n_ready; ++j) | |
3051 | if (INSN_LAST_RFS_WIN (p[j]) == i) | |
3052 | fprintf (sched_dump, " %s", | |
3053 | (*current_sched_info->print_insn) (p[j], 0)); | |
3054 | } | |
3055 | fprintf (sched_dump, "\n"); | |
3056 | } | |
4bd12f3d MK |
3057 | } |
3058 | ||
696d5fa1 MK |
3059 | /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end |
3060 | of array. */ | |
3061 | static void | |
3062 | ready_sort_debug (struct ready_list *ready) | |
176f9a7b | 3063 | { |
ce18efcb | 3064 | int i; |
ce1ce33a | 3065 | rtx_insn **first = ready_lastpos (ready); |
ce18efcb | 3066 | |
63387a85 | 3067 | for (i = 0; i < ready->n_ready; ++i) |
696d5fa1 MK |
3068 | if (!DEBUG_INSN_P (first[i])) |
3069 | INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i; | |
63387a85 | 3070 | |
696d5fa1 MK |
3071 | qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug); |
3072 | } | |
63387a85 | 3073 | |
696d5fa1 MK |
3074 | /* Sort non-debug insns in the ready list READY by ascending priority. |
3075 | Assumes that all debug insns are separated from the real insns. */ | |
3076 | static void | |
3077 | ready_sort_real (struct ready_list *ready) | |
3078 | { | |
3079 | int i; | |
3080 | rtx_insn **first = ready_lastpos (ready); | |
3081 | int n_ready_real = ready->n_ready - ready->n_debug; | |
3082 | ||
3083 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) | |
3084 | for (i = 0; i < n_ready_real; ++i) | |
3085 | setup_insn_reg_pressure_info (first[i]); | |
3086 | else if (sched_pressure == SCHED_PRESSURE_MODEL | |
3087 | && model_curr_point < model_num_insns) | |
3088 | model_set_excess_costs (first, n_ready_real); | |
4bd12f3d MK |
3089 | |
3090 | rank_for_schedule_stats_t stats1; | |
3091 | if (sched_verbose >= 4) | |
3092 | stats1 = rank_for_schedule_stats; | |
3093 | ||
696d5fa1 MK |
3094 | if (n_ready_real == 2) |
3095 | swap_sort (first, n_ready_real); | |
3096 | else if (n_ready_real > 2) | |
f9f43fb3 | 3097 | qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule); |
4bd12f3d MK |
3098 | |
3099 | if (sched_verbose >= 4) | |
3100 | { | |
3101 | rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats); | |
b75ae7f4 | 3102 | print_rank_for_schedule_stats (";;\t\t", &stats1, ready); |
4bd12f3d | 3103 | } |
8c660648 JL |
3104 | } |
3105 | ||
696d5fa1 MK |
3106 | /* Sort the ready list READY by ascending priority. */ |
3107 | static void | |
3108 | ready_sort (struct ready_list *ready) | |
3109 | { | |
3110 | if (ready->n_debug > 0) | |
3111 | ready_sort_debug (ready); | |
3112 | else | |
3113 | ready_sort_real (ready); | |
3114 | } | |
3115 | ||
8c660648 | 3116 | /* PREV is an insn that is ready to execute. Adjust its priority if that |
c46a37c4 | 3117 | will help shorten or lengthen register lifetimes as appropriate. Also |
fa10beec | 3118 | provide a hook for the target to tweak itself. */ |
8c660648 | 3119 | |
cbb13457 | 3120 | HAIFA_INLINE static void |
f57aa6b0 | 3121 | adjust_priority (rtx_insn *prev) |
8c660648 | 3122 | { |
c46a37c4 RH |
3123 | /* ??? There used to be code here to try and estimate how an insn |
3124 | affected register lifetimes, but it did it by looking at REG_DEAD | |
7a403706 | 3125 | notes, which we removed in schedule_region. Nor did it try to |
c46a37c4 | 3126 | take into account register pressure or anything useful like that. |
8c660648 | 3127 | |
c46a37c4 | 3128 | Revisit when we have a machine model to work with and not before. */ |
197043f5 | 3129 | |
c237e94a ZW |
3130 | if (targetm.sched.adjust_priority) |
3131 | INSN_PRIORITY (prev) = | |
5fd9b178 | 3132 | targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev)); |
8c660648 JL |
3133 | } |
3134 | ||
e855c69d AB |
3135 | /* Advance DFA state STATE on one cycle. */ |
3136 | void | |
3137 | advance_state (state_t state) | |
fae15c93 | 3138 | { |
1c3d0d93 MK |
3139 | if (targetm.sched.dfa_pre_advance_cycle) |
3140 | targetm.sched.dfa_pre_advance_cycle (); | |
3141 | ||
fa0aee89 | 3142 | if (targetm.sched.dfa_pre_cycle_insn) |
e855c69d | 3143 | state_transition (state, |
fa0aee89 PB |
3144 | targetm.sched.dfa_pre_cycle_insn ()); |
3145 | ||
e855c69d | 3146 | state_transition (state, NULL); |
b8698a0f | 3147 | |
fa0aee89 | 3148 | if (targetm.sched.dfa_post_cycle_insn) |
e855c69d | 3149 | state_transition (state, |
fa0aee89 | 3150 | targetm.sched.dfa_post_cycle_insn ()); |
1c3d0d93 MK |
3151 | |
3152 | if (targetm.sched.dfa_post_advance_cycle) | |
3153 | targetm.sched.dfa_post_advance_cycle (); | |
fae15c93 VM |
3154 | } |
3155 | ||
e855c69d AB |
3156 | /* Advance time on one cycle. */ |
3157 | HAIFA_INLINE static void | |
3158 | advance_one_cycle (void) | |
3159 | { | |
3160 | advance_state (curr_state); | |
d5c67efd | 3161 | if (sched_verbose >= 4) |
f50d71ec | 3162 | fprintf (sched_dump, ";;\tAdvance the current state.\n"); |
e855c69d AB |
3163 | } |
3164 | ||
ce18efcb VM |
3165 | /* Update register pressure after scheduling INSN. */ |
3166 | static void | |
f57aa6b0 | 3167 | update_register_pressure (rtx_insn *insn) |
ce18efcb VM |
3168 | { |
3169 | struct reg_use_data *use; | |
3170 | struct reg_set_data *set; | |
3171 | ||
c11b0b3b AO |
3172 | gcc_checking_assert (!DEBUG_INSN_P (insn)); |
3173 | ||
ce18efcb | 3174 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) |
7a1aca9c RS |
3175 | if (dying_use_p (use)) |
3176 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, | |
3177 | use->regno, false); | |
ce18efcb | 3178 | for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set) |
7a1aca9c RS |
3179 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, |
3180 | set->regno, true); | |
ce18efcb VM |
3181 | } |
3182 | ||
3183 | /* Set up or update (if UPDATE_P) max register pressure (see its | |
3184 | meaning in sched-int.h::_haifa_insn_data) for all current BB insns | |
3185 | after insn AFTER. */ | |
3186 | static void | |
dc01c3d1 | 3187 | setup_insn_max_reg_pressure (rtx_insn *after, bool update_p) |
ce18efcb VM |
3188 | { |
3189 | int i, p; | |
3190 | bool eq_p; | |
f57aa6b0 | 3191 | rtx_insn *insn; |
ce18efcb VM |
3192 | static int max_reg_pressure[N_REG_CLASSES]; |
3193 | ||
3194 | save_reg_pressure (); | |
1756cb66 VM |
3195 | for (i = 0; i < ira_pressure_classes_num; i++) |
3196 | max_reg_pressure[ira_pressure_classes[i]] | |
3197 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb | 3198 | for (insn = NEXT_INSN (after); |
0495bafc VM |
3199 | insn != NULL_RTX && ! BARRIER_P (insn) |
3200 | && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after); | |
ce18efcb VM |
3201 | insn = NEXT_INSN (insn)) |
3202 | if (NONDEBUG_INSN_P (insn)) | |
3203 | { | |
3204 | eq_p = true; | |
1756cb66 | 3205 | for (i = 0; i < ira_pressure_classes_num; i++) |
ce18efcb | 3206 | { |
1756cb66 | 3207 | p = max_reg_pressure[ira_pressure_classes[i]]; |
ce18efcb VM |
3208 | if (INSN_MAX_REG_PRESSURE (insn)[i] != p) |
3209 | { | |
3210 | eq_p = false; | |
3211 | INSN_MAX_REG_PRESSURE (insn)[i] | |
1756cb66 | 3212 | = max_reg_pressure[ira_pressure_classes[i]]; |
ce18efcb VM |
3213 | } |
3214 | } | |
3215 | if (update_p && eq_p) | |
3216 | break; | |
3217 | update_register_pressure (insn); | |
1756cb66 VM |
3218 | for (i = 0; i < ira_pressure_classes_num; i++) |
3219 | if (max_reg_pressure[ira_pressure_classes[i]] | |
3220 | < curr_reg_pressure[ira_pressure_classes[i]]) | |
3221 | max_reg_pressure[ira_pressure_classes[i]] | |
3222 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb VM |
3223 | } |
3224 | restore_reg_pressure (); | |
3225 | } | |
3226 | ||
3227 | /* Update the current register pressure after scheduling INSN. Update | |
3228 | also max register pressure for unscheduled insns of the current | |
3229 | BB. */ | |
3230 | static void | |
f57aa6b0 | 3231 | update_reg_and_insn_max_reg_pressure (rtx_insn *insn) |
ce18efcb VM |
3232 | { |
3233 | int i; | |
3234 | int before[N_REG_CLASSES]; | |
3235 | ||
1756cb66 VM |
3236 | for (i = 0; i < ira_pressure_classes_num; i++) |
3237 | before[i] = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb | 3238 | update_register_pressure (insn); |
1756cb66 VM |
3239 | for (i = 0; i < ira_pressure_classes_num; i++) |
3240 | if (curr_reg_pressure[ira_pressure_classes[i]] != before[i]) | |
ce18efcb | 3241 | break; |
1756cb66 | 3242 | if (i < ira_pressure_classes_num) |
ce18efcb VM |
3243 | setup_insn_max_reg_pressure (insn, true); |
3244 | } | |
3245 | ||
3246 | /* Set up register pressure at the beginning of basic block BB whose | |
3247 | insns starting after insn AFTER. Set up also max register pressure | |
3248 | for all insns of the basic block. */ | |
3249 | void | |
dc01c3d1 | 3250 | sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after) |
ce18efcb | 3251 | { |
60867e8c | 3252 | gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED); |
ce18efcb VM |
3253 | initiate_bb_reg_pressure_info (bb); |
3254 | setup_insn_max_reg_pressure (after, false); | |
3255 | } | |
51564684 | 3256 | \f |
e2724e63 BS |
3257 | /* If doing predication while scheduling, verify whether INSN, which |
3258 | has just been scheduled, clobbers the conditions of any | |
3259 | instructions that must be predicated in order to break their | |
3260 | dependencies. If so, remove them from the queues so that they will | |
3261 | only be scheduled once their control dependency is resolved. */ | |
3262 | ||
3263 | static void | |
90831096 | 3264 | check_clobbered_conditions (rtx_insn *insn) |
e2724e63 BS |
3265 | { |
3266 | HARD_REG_SET t; | |
3267 | int i; | |
3268 | ||
3269 | if ((current_sched_info->flags & DO_PREDICATION) == 0) | |
3270 | return; | |
3271 | ||
356bf593 | 3272 | find_all_hard_reg_sets (insn, &t, true); |
e2724e63 BS |
3273 | |
3274 | restart: | |
3275 | for (i = 0; i < ready.n_ready; i++) | |
3276 | { | |
ce1ce33a | 3277 | rtx_insn *x = ready_element (&ready, i); |
e2724e63 BS |
3278 | if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t)) |
3279 | { | |
3280 | ready_remove_insn (x); | |
3281 | goto restart; | |
3282 | } | |
3283 | } | |
3284 | for (i = 0; i <= max_insn_queue_index; i++) | |
3285 | { | |
96b70414 | 3286 | rtx_insn_list *link; |
e2724e63 BS |
3287 | int q = NEXT_Q_AFTER (q_ptr, i); |
3288 | ||
3289 | restart_queue: | |
96b70414 | 3290 | for (link = insn_queue[q]; link; link = link->next ()) |
e2724e63 | 3291 | { |
96b70414 | 3292 | rtx_insn *x = link->insn (); |
e2724e63 BS |
3293 | if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t)) |
3294 | { | |
3295 | queue_remove (x); | |
3296 | goto restart_queue; | |
3297 | } | |
3298 | } | |
3299 | } | |
3300 | } | |
3301 | \f | |
7a1aca9c RS |
3302 | /* Return (in order): |
3303 | ||
3304 | - positive if INSN adversely affects the pressure on one | |
3305 | register class | |
3306 | ||
3307 | - negative if INSN reduces the pressure on one register class | |
3308 | ||
3309 | - 0 if INSN doesn't affect the pressure on any register class. */ | |
3310 | ||
3311 | static int | |
3312 | model_classify_pressure (struct model_insn_info *insn) | |
3313 | { | |
3314 | struct reg_pressure_data *reg_pressure; | |
3315 | int death[N_REG_CLASSES]; | |
3316 | int pci, cl, sum; | |
3317 | ||
3318 | calculate_reg_deaths (insn->insn, death); | |
3319 | reg_pressure = INSN_REG_PRESSURE (insn->insn); | |
3320 | sum = 0; | |
3321 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3322 | { | |
3323 | cl = ira_pressure_classes[pci]; | |
3324 | if (death[cl] < reg_pressure[pci].set_increase) | |
3325 | return 1; | |
3326 | sum += reg_pressure[pci].set_increase - death[cl]; | |
3327 | } | |
3328 | return sum; | |
3329 | } | |
3330 | ||
3331 | /* Return true if INSN1 should come before INSN2 in the model schedule. */ | |
3332 | ||
3333 | static int | |
3334 | model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2) | |
3335 | { | |
3336 | unsigned int height1, height2; | |
3337 | unsigned int priority1, priority2; | |
3338 | ||
3339 | /* Prefer instructions with a higher model priority. */ | |
3340 | if (insn1->model_priority != insn2->model_priority) | |
3341 | return insn1->model_priority > insn2->model_priority; | |
3342 | ||
3343 | /* Combine the length of the longest path of satisfied true dependencies | |
3344 | that leads to each instruction (depth) with the length of the longest | |
3345 | path of any dependencies that leads from the instruction (alap). | |
3346 | Prefer instructions with the greatest combined length. If the combined | |
3347 | lengths are equal, prefer instructions with the greatest depth. | |
3348 | ||
3349 | The idea is that, if we have a set S of "equal" instructions that each | |
3350 | have ALAP value X, and we pick one such instruction I, any true-dependent | |
3351 | successors of I that have ALAP value X - 1 should be preferred over S. | |
3352 | This encourages the schedule to be "narrow" rather than "wide". | |
3353 | However, if I is a low-priority instruction that we decided to | |
3354 | schedule because of its model_classify_pressure, and if there | |
3355 | is a set of higher-priority instructions T, the aforementioned | |
3356 | successors of I should not have the edge over T. */ | |
3357 | height1 = insn1->depth + insn1->alap; | |
3358 | height2 = insn2->depth + insn2->alap; | |
3359 | if (height1 != height2) | |
3360 | return height1 > height2; | |
3361 | if (insn1->depth != insn2->depth) | |
3362 | return insn1->depth > insn2->depth; | |
3363 | ||
3364 | /* We have no real preference between INSN1 an INSN2 as far as attempts | |
3365 | to reduce pressure go. Prefer instructions with higher priorities. */ | |
3366 | priority1 = INSN_PRIORITY (insn1->insn); | |
3367 | priority2 = INSN_PRIORITY (insn2->insn); | |
3368 | if (priority1 != priority2) | |
3369 | return priority1 > priority2; | |
3370 | ||
3371 | /* Use the original rtl sequence as a tie-breaker. */ | |
3372 | return insn1 < insn2; | |
3373 | } | |
3374 | ||
3375 | /* Add INSN to the model worklist immediately after PREV. Add it to the | |
3376 | beginning of the list if PREV is null. */ | |
3377 | ||
3378 | static void | |
3379 | model_add_to_worklist_at (struct model_insn_info *insn, | |
3380 | struct model_insn_info *prev) | |
3381 | { | |
3382 | gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE); | |
3383 | QUEUE_INDEX (insn->insn) = QUEUE_READY; | |
3384 | ||
3385 | insn->prev = prev; | |
3386 | if (prev) | |
3387 | { | |
3388 | insn->next = prev->next; | |
3389 | prev->next = insn; | |
3390 | } | |
3391 | else | |
3392 | { | |
3393 | insn->next = model_worklist; | |
3394 | model_worklist = insn; | |
3395 | } | |
3396 | if (insn->next) | |
3397 | insn->next->prev = insn; | |
3398 | } | |
3399 | ||
3400 | /* Remove INSN from the model worklist. */ | |
3401 | ||
3402 | static void | |
3403 | model_remove_from_worklist (struct model_insn_info *insn) | |
3404 | { | |
3405 | gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY); | |
3406 | QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE; | |
3407 | ||
3408 | if (insn->prev) | |
3409 | insn->prev->next = insn->next; | |
3410 | else | |
3411 | model_worklist = insn->next; | |
3412 | if (insn->next) | |
3413 | insn->next->prev = insn->prev; | |
3414 | } | |
3415 | ||
3416 | /* Add INSN to the model worklist. Start looking for a suitable position | |
3417 | between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS | |
3418 | insns either side. A null PREV indicates the beginning of the list and | |
3419 | a null NEXT indicates the end. */ | |
3420 | ||
3421 | static void | |
3422 | model_add_to_worklist (struct model_insn_info *insn, | |
3423 | struct model_insn_info *prev, | |
3424 | struct model_insn_info *next) | |
3425 | { | |
3426 | int count; | |
3427 | ||
3428 | count = MAX_SCHED_READY_INSNS; | |
3429 | if (count > 0 && prev && model_order_p (insn, prev)) | |
3430 | do | |
3431 | { | |
3432 | count--; | |
3433 | prev = prev->prev; | |
3434 | } | |
3435 | while (count > 0 && prev && model_order_p (insn, prev)); | |
3436 | else | |
3437 | while (count > 0 && next && model_order_p (next, insn)) | |
3438 | { | |
3439 | count--; | |
3440 | prev = next; | |
3441 | next = next->next; | |
3442 | } | |
3443 | model_add_to_worklist_at (insn, prev); | |
3444 | } | |
3445 | ||
3446 | /* INSN may now have a higher priority (in the model_order_p sense) | |
3447 | than before. Move it up the worklist if necessary. */ | |
3448 | ||
3449 | static void | |
3450 | model_promote_insn (struct model_insn_info *insn) | |
3451 | { | |
3452 | struct model_insn_info *prev; | |
3453 | int count; | |
3454 | ||
3455 | prev = insn->prev; | |
3456 | count = MAX_SCHED_READY_INSNS; | |
3457 | while (count > 0 && prev && model_order_p (insn, prev)) | |
3458 | { | |
3459 | count--; | |
3460 | prev = prev->prev; | |
3461 | } | |
3462 | if (prev != insn->prev) | |
3463 | { | |
3464 | model_remove_from_worklist (insn); | |
3465 | model_add_to_worklist_at (insn, prev); | |
3466 | } | |
3467 | } | |
3468 | ||
3469 | /* Add INSN to the end of the model schedule. */ | |
3470 | ||
3471 | static void | |
f57aa6b0 | 3472 | model_add_to_schedule (rtx_insn *insn) |
7a1aca9c RS |
3473 | { |
3474 | unsigned int point; | |
3475 | ||
3476 | gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE); | |
3477 | QUEUE_INDEX (insn) = QUEUE_SCHEDULED; | |
3478 | ||
9771b263 DN |
3479 | point = model_schedule.length (); |
3480 | model_schedule.quick_push (insn); | |
7a1aca9c RS |
3481 | INSN_MODEL_INDEX (insn) = point + 1; |
3482 | } | |
3483 | ||
3484 | /* Analyze the instructions that are to be scheduled, setting up | |
3485 | MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready | |
3486 | instructions to model_worklist. */ | |
3487 | ||
3488 | static void | |
3489 | model_analyze_insns (void) | |
3490 | { | |
40a243d4 | 3491 | rtx_insn *start, *end, *iter; |
7a1aca9c RS |
3492 | sd_iterator_def sd_it; |
3493 | dep_t dep; | |
3494 | struct model_insn_info *insn, *con; | |
3495 | ||
3496 | model_num_insns = 0; | |
3497 | start = PREV_INSN (current_sched_info->next_tail); | |
3498 | end = current_sched_info->prev_head; | |
3499 | for (iter = start; iter != end; iter = PREV_INSN (iter)) | |
3500 | if (NONDEBUG_INSN_P (iter)) | |
3501 | { | |
3502 | insn = MODEL_INSN_INFO (iter); | |
3503 | insn->insn = iter; | |
3504 | FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep) | |
3505 | { | |
3506 | con = MODEL_INSN_INFO (DEP_CON (dep)); | |
3507 | if (con->insn && insn->alap < con->alap + 1) | |
3508 | insn->alap = con->alap + 1; | |
3509 | } | |
3510 | ||
3511 | insn->old_queue = QUEUE_INDEX (iter); | |
3512 | QUEUE_INDEX (iter) = QUEUE_NOWHERE; | |
3513 | ||
3514 | insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK); | |
3515 | if (insn->unscheduled_preds == 0) | |
3516 | model_add_to_worklist (insn, NULL, model_worklist); | |
3517 | ||
3518 | model_num_insns++; | |
3519 | } | |
3520 | } | |
3521 | ||
3522 | /* The global state describes the register pressure at the start of the | |
3523 | model schedule. Initialize GROUP accordingly. */ | |
3524 | ||
3525 | static void | |
3526 | model_init_pressure_group (struct model_pressure_group *group) | |
3527 | { | |
3528 | int pci, cl; | |
3529 | ||
3530 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3531 | { | |
3532 | cl = ira_pressure_classes[pci]; | |
3533 | group->limits[pci].pressure = curr_reg_pressure[cl]; | |
3534 | group->limits[pci].point = 0; | |
3535 | } | |
3536 | /* Use index model_num_insns to record the state after the last | |
3537 | instruction in the model schedule. */ | |
3538 | group->model = XNEWVEC (struct model_pressure_data, | |
3539 | (model_num_insns + 1) * ira_pressure_classes_num); | |
3540 | } | |
3541 | ||
3542 | /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE. | |
3543 | Update the maximum pressure for the whole schedule. */ | |
3544 | ||
3545 | static void | |
3546 | model_record_pressure (struct model_pressure_group *group, | |
3547 | int point, int pci, int pressure) | |
3548 | { | |
3549 | MODEL_REF_PRESSURE (group, point, pci) = pressure; | |
3550 | if (group->limits[pci].pressure < pressure) | |
3551 | { | |
3552 | group->limits[pci].pressure = pressure; | |
3553 | group->limits[pci].point = point; | |
3554 | } | |
3555 | } | |
3556 | ||
3557 | /* INSN has just been added to the end of the model schedule. Record its | |
3558 | register-pressure information. */ | |
3559 | ||
3560 | static void | |
3561 | model_record_pressures (struct model_insn_info *insn) | |
3562 | { | |
3563 | struct reg_pressure_data *reg_pressure; | |
3564 | int point, pci, cl, delta; | |
3565 | int death[N_REG_CLASSES]; | |
3566 | ||
3567 | point = model_index (insn->insn); | |
3568 | if (sched_verbose >= 2) | |
3569 | { | |
7a1aca9c RS |
3570 | if (point == 0) |
3571 | { | |
3572 | fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n"); | |
3573 | fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n"); | |
3574 | } | |
7a1aca9c RS |
3575 | fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ", |
3576 | point, INSN_UID (insn->insn), insn->model_priority, | |
3577 | insn->depth + insn->alap, insn->depth, | |
7eba871a SB |
3578 | INSN_PRIORITY (insn->insn), |
3579 | str_pattern_slim (PATTERN (insn->insn))); | |
7a1aca9c RS |
3580 | } |
3581 | calculate_reg_deaths (insn->insn, death); | |
3582 | reg_pressure = INSN_REG_PRESSURE (insn->insn); | |
3583 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3584 | { | |
3585 | cl = ira_pressure_classes[pci]; | |
3586 | delta = reg_pressure[pci].set_increase - death[cl]; | |
3587 | if (sched_verbose >= 2) | |
3588 | fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl], | |
3589 | curr_reg_pressure[cl], delta); | |
3590 | model_record_pressure (&model_before_pressure, point, pci, | |
3591 | curr_reg_pressure[cl]); | |
3592 | } | |
3593 | if (sched_verbose >= 2) | |
3594 | fprintf (sched_dump, "\n"); | |
3595 | } | |
3596 | ||
3597 | /* All instructions have been added to the model schedule. Record the | |
3598 | final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */ | |
3599 | ||
3600 | static void | |
3601 | model_record_final_pressures (struct model_pressure_group *group) | |
3602 | { | |
3603 | int point, pci, max_pressure, ref_pressure, cl; | |
3604 | ||
3605 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3606 | { | |
3607 | /* Record the final pressure for this class. */ | |
3608 | cl = ira_pressure_classes[pci]; | |
3609 | point = model_num_insns; | |
3610 | ref_pressure = curr_reg_pressure[cl]; | |
3611 | model_record_pressure (group, point, pci, ref_pressure); | |
3612 | ||
3613 | /* Record the original maximum pressure. */ | |
3614 | group->limits[pci].orig_pressure = group->limits[pci].pressure; | |
3615 | ||
3616 | /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */ | |
3617 | max_pressure = ref_pressure; | |
3618 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
3619 | while (point > 0) | |
3620 | { | |
3621 | point--; | |
3622 | ref_pressure = MODEL_REF_PRESSURE (group, point, pci); | |
3623 | max_pressure = MAX (max_pressure, ref_pressure); | |
3624 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
3625 | } | |
3626 | } | |
3627 | } | |
3628 | ||
3629 | /* Update all successors of INSN, given that INSN has just been scheduled. */ | |
3630 | ||
3631 | static void | |
3632 | model_add_successors_to_worklist (struct model_insn_info *insn) | |
3633 | { | |
3634 | sd_iterator_def sd_it; | |
3635 | struct model_insn_info *con; | |
3636 | dep_t dep; | |
3637 | ||
3638 | FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep) | |
3639 | { | |
3640 | con = MODEL_INSN_INFO (DEP_CON (dep)); | |
3641 | /* Ignore debug instructions, and instructions from other blocks. */ | |
3642 | if (con->insn) | |
3643 | { | |
3644 | con->unscheduled_preds--; | |
3645 | ||
3646 | /* Update the depth field of each true-dependent successor. | |
3647 | Increasing the depth gives them a higher priority than | |
3648 | before. */ | |
3649 | if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1) | |
3650 | { | |
3651 | con->depth = insn->depth + 1; | |
3652 | if (QUEUE_INDEX (con->insn) == QUEUE_READY) | |
3653 | model_promote_insn (con); | |
3654 | } | |
3655 | ||
3656 | /* If this is a true dependency, or if there are no remaining | |
3657 | dependencies for CON (meaning that CON only had non-true | |
3658 | dependencies), make sure that CON is on the worklist. | |
3659 | We don't bother otherwise because it would tend to fill the | |
3660 | worklist with a lot of low-priority instructions that are not | |
3661 | yet ready to issue. */ | |
3662 | if ((con->depth > 0 || con->unscheduled_preds == 0) | |
3663 | && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE) | |
3664 | model_add_to_worklist (con, insn, insn->next); | |
3665 | } | |
3666 | } | |
3667 | } | |
3668 | ||
3669 | /* Give INSN a higher priority than any current instruction, then give | |
3670 | unscheduled predecessors of INSN a higher priority still. If any of | |
3671 | those predecessors are not on the model worklist, do the same for its | |
3672 | predecessors, and so on. */ | |
3673 | ||
3674 | static void | |
3675 | model_promote_predecessors (struct model_insn_info *insn) | |
3676 | { | |
3677 | struct model_insn_info *pro, *first; | |
3678 | sd_iterator_def sd_it; | |
3679 | dep_t dep; | |
3680 | ||
3681 | if (sched_verbose >= 7) | |
3682 | fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of", | |
3683 | INSN_UID (insn->insn), model_next_priority); | |
3684 | insn->model_priority = model_next_priority++; | |
3685 | model_remove_from_worklist (insn); | |
3686 | model_add_to_worklist_at (insn, NULL); | |
3687 | ||
3688 | first = NULL; | |
3689 | for (;;) | |
3690 | { | |
3691 | FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep) | |
3692 | { | |
3693 | pro = MODEL_INSN_INFO (DEP_PRO (dep)); | |
3694 | /* The first test is to ignore debug instructions, and instructions | |
3695 | from other blocks. */ | |
3696 | if (pro->insn | |
3697 | && pro->model_priority != model_next_priority | |
3698 | && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED) | |
3699 | { | |
3700 | pro->model_priority = model_next_priority; | |
3701 | if (sched_verbose >= 7) | |
3702 | fprintf (sched_dump, " %d", INSN_UID (pro->insn)); | |
3703 | if (QUEUE_INDEX (pro->insn) == QUEUE_READY) | |
3704 | { | |
3705 | /* PRO is already in the worklist, but it now has | |
3706 | a higher priority than before. Move it at the | |
3707 | appropriate place. */ | |
3708 | model_remove_from_worklist (pro); | |
3709 | model_add_to_worklist (pro, NULL, model_worklist); | |
3710 | } | |
3711 | else | |
3712 | { | |
3713 | /* PRO isn't in the worklist. Recursively process | |
3714 | its predecessors until we find one that is. */ | |
3715 | pro->next = first; | |
3716 | first = pro; | |
3717 | } | |
3718 | } | |
3719 | } | |
3720 | if (!first) | |
3721 | break; | |
3722 | insn = first; | |
3723 | first = insn->next; | |
3724 | } | |
3725 | if (sched_verbose >= 7) | |
3726 | fprintf (sched_dump, " = %d\n", model_next_priority); | |
3727 | model_next_priority++; | |
3728 | } | |
3729 | ||
3730 | /* Pick one instruction from model_worklist and process it. */ | |
3731 | ||
3732 | static void | |
3733 | model_choose_insn (void) | |
3734 | { | |
3735 | struct model_insn_info *insn, *fallback; | |
3736 | int count; | |
3737 | ||
3738 | if (sched_verbose >= 7) | |
3739 | { | |
3740 | fprintf (sched_dump, ";;\t+--- worklist:\n"); | |
3741 | insn = model_worklist; | |
3742 | count = MAX_SCHED_READY_INSNS; | |
3743 | while (count > 0 && insn) | |
3744 | { | |
3745 | fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d]\n", | |
3746 | INSN_UID (insn->insn), insn->model_priority, | |
3747 | insn->depth + insn->alap, insn->depth, | |
3748 | INSN_PRIORITY (insn->insn)); | |
3749 | count--; | |
3750 | insn = insn->next; | |
3751 | } | |
3752 | } | |
3753 | ||
3754 | /* Look for a ready instruction whose model_classify_priority is zero | |
3755 | or negative, picking the highest-priority one. Adding such an | |
3756 | instruction to the schedule now should do no harm, and may actually | |
3757 | do some good. | |
3758 | ||
3759 | Failing that, see whether there is an instruction with the highest | |
3760 | extant model_priority that is not yet ready, but which would reduce | |
3761 | pressure if it became ready. This is designed to catch cases like: | |
3762 | ||
3763 | (set (mem (reg R1)) (reg R2)) | |
3764 | ||
3765 | where the instruction is the last remaining use of R1 and where the | |
3766 | value of R2 is not yet available (or vice versa). The death of R1 | |
3767 | means that this instruction already reduces pressure. It is of | |
3768 | course possible that the computation of R2 involves other registers | |
3769 | that are hard to kill, but such cases are rare enough for this | |
3770 | heuristic to be a win in general. | |
3771 | ||
3772 | Failing that, just pick the highest-priority instruction in the | |
3773 | worklist. */ | |
3774 | count = MAX_SCHED_READY_INSNS; | |
3775 | insn = model_worklist; | |
3776 | fallback = 0; | |
3777 | for (;;) | |
3778 | { | |
3779 | if (count == 0 || !insn) | |
3780 | { | |
3781 | insn = fallback ? fallback : model_worklist; | |
3782 | break; | |
3783 | } | |
3784 | if (insn->unscheduled_preds) | |
3785 | { | |
3786 | if (model_worklist->model_priority == insn->model_priority | |
3787 | && !fallback | |
3788 | && model_classify_pressure (insn) < 0) | |
3789 | fallback = insn; | |
3790 | } | |
3791 | else | |
3792 | { | |
3793 | if (model_classify_pressure (insn) <= 0) | |
3794 | break; | |
3795 | } | |
3796 | count--; | |
3797 | insn = insn->next; | |
3798 | } | |
3799 | ||
3800 | if (sched_verbose >= 7 && insn != model_worklist) | |
3801 | { | |
3802 | if (insn->unscheduled_preds) | |
3803 | fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n", | |
3804 | INSN_UID (insn->insn)); | |
3805 | else | |
3806 | fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n", | |
3807 | INSN_UID (insn->insn)); | |
3808 | } | |
3809 | if (insn->unscheduled_preds) | |
3810 | /* INSN isn't yet ready to issue. Give all its predecessors the | |
3811 | highest priority. */ | |
3812 | model_promote_predecessors (insn); | |
3813 | else | |
3814 | { | |
3815 | /* INSN is ready. Add it to the end of model_schedule and | |
3816 | process its successors. */ | |
3817 | model_add_successors_to_worklist (insn); | |
3818 | model_remove_from_worklist (insn); | |
3819 | model_add_to_schedule (insn->insn); | |
3820 | model_record_pressures (insn); | |
3821 | update_register_pressure (insn->insn); | |
3822 | } | |
3823 | } | |
3824 | ||
3825 | /* Restore all QUEUE_INDEXs to the values that they had before | |
3826 | model_start_schedule was called. */ | |
3827 | ||
3828 | static void | |
3829 | model_reset_queue_indices (void) | |
3830 | { | |
3831 | unsigned int i; | |
f57aa6b0 | 3832 | rtx_insn *insn; |
7a1aca9c | 3833 | |
9771b263 | 3834 | FOR_EACH_VEC_ELT (model_schedule, i, insn) |
7a1aca9c RS |
3835 | QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue; |
3836 | } | |
3837 | ||
3838 | /* We have calculated the model schedule and spill costs. Print a summary | |
3839 | to sched_dump. */ | |
3840 | ||
3841 | static void | |
3842 | model_dump_pressure_summary (void) | |
3843 | { | |
3844 | int pci, cl; | |
3845 | ||
3846 | fprintf (sched_dump, ";; Pressure summary:"); | |
3847 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3848 | { | |
3849 | cl = ira_pressure_classes[pci]; | |
3850 | fprintf (sched_dump, " %s:%d", reg_class_names[cl], | |
3851 | model_before_pressure.limits[pci].pressure); | |
3852 | } | |
3853 | fprintf (sched_dump, "\n\n"); | |
3854 | } | |
3855 | ||
3856 | /* Initialize the SCHED_PRESSURE_MODEL information for the current | |
3857 | scheduling region. */ | |
3858 | ||
3859 | static void | |
4dd9ac6c | 3860 | model_start_schedule (basic_block bb) |
7a1aca9c | 3861 | { |
7a1aca9c | 3862 | model_next_priority = 1; |
9771b263 | 3863 | model_schedule.create (sched_max_luid); |
7a1aca9c RS |
3864 | model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid); |
3865 | ||
4dd9ac6c | 3866 | gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head))); |
7a1aca9c RS |
3867 | initiate_reg_pressure_info (df_get_live_in (bb)); |
3868 | ||
3869 | model_analyze_insns (); | |
3870 | model_init_pressure_group (&model_before_pressure); | |
3871 | while (model_worklist) | |
3872 | model_choose_insn (); | |
9771b263 | 3873 | gcc_assert (model_num_insns == (int) model_schedule.length ()); |
7a1aca9c RS |
3874 | if (sched_verbose >= 2) |
3875 | fprintf (sched_dump, "\n"); | |
3876 | ||
3877 | model_record_final_pressures (&model_before_pressure); | |
3878 | model_reset_queue_indices (); | |
3879 | ||
3880 | XDELETEVEC (model_insns); | |
3881 | ||
3882 | model_curr_point = 0; | |
3883 | initiate_reg_pressure_info (df_get_live_in (bb)); | |
3884 | if (sched_verbose >= 1) | |
3885 | model_dump_pressure_summary (); | |
3886 | } | |
3887 | ||
3888 | /* Free the information associated with GROUP. */ | |
3889 | ||
3890 | static void | |
3891 | model_finalize_pressure_group (struct model_pressure_group *group) | |
3892 | { | |
3893 | XDELETEVEC (group->model); | |
3894 | } | |
3895 | ||
3896 | /* Free the information created by model_start_schedule. */ | |
3897 | ||
3898 | static void | |
3899 | model_end_schedule (void) | |
3900 | { | |
3901 | model_finalize_pressure_group (&model_before_pressure); | |
9771b263 | 3902 | model_schedule.release (); |
7a1aca9c | 3903 | } |
4dd9ac6c MK |
3904 | |
3905 | /* Prepare reg pressure scheduling for basic block BB. */ | |
3906 | static void | |
3907 | sched_pressure_start_bb (basic_block bb) | |
3908 | { | |
3909 | /* Set the number of available registers for each class taking into account | |
3910 | relative probability of current basic block versus function prologue and | |
3911 | epilogue. | |
3912 | * If the basic block executes much more often than the prologue/epilogue | |
3913 | (e.g., inside a hot loop), then cost of spill in the prologue is close to | |
3914 | nil, so the effective number of available registers is | |
6d5c37d0 | 3915 | (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0). |
4dd9ac6c MK |
3916 | * If the basic block executes as often as the prologue/epilogue, |
3917 | then spill in the block is as costly as in the prologue, so the effective | |
3918 | number of available registers is | |
6d5c37d0 PH |
3919 | (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] |
3920 | - call_saved_regs_num[cl]). | |
4dd9ac6c MK |
3921 | Note that all-else-equal, we prefer to spill in the prologue, since that |
3922 | allows "extra" registers for other basic blocks of the function. | |
3923 | * If the basic block is on the cold path of the function and executes | |
3924 | rarely, then we should always prefer to spill in the block, rather than | |
3925 | in the prologue/epilogue. The effective number of available register is | |
6d5c37d0 PH |
3926 | (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] |
3927 | - call_saved_regs_num[cl]). */ | |
4dd9ac6c MK |
3928 | { |
3929 | int i; | |
e7a74006 JH |
3930 | int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun); |
3931 | int bb_freq = bb->count.to_frequency (cfun); | |
4dd9ac6c MK |
3932 | |
3933 | if (bb_freq == 0) | |
3934 | { | |
3935 | if (entry_freq == 0) | |
3936 | entry_freq = bb_freq = 1; | |
3937 | } | |
3938 | if (bb_freq < entry_freq) | |
3939 | bb_freq = entry_freq; | |
3940 | ||
3941 | for (i = 0; i < ira_pressure_classes_num; ++i) | |
3942 | { | |
3943 | enum reg_class cl = ira_pressure_classes[i]; | |
6d5c37d0 PH |
3944 | sched_class_regs_num[cl] = ira_class_hard_regs_num[cl] |
3945 | - fixed_regs_num[cl]; | |
4dd9ac6c | 3946 | sched_class_regs_num[cl] |
6d5c37d0 | 3947 | -= (call_saved_regs_num[cl] * entry_freq) / bb_freq; |
4dd9ac6c MK |
3948 | } |
3949 | } | |
3950 | ||
3951 | if (sched_pressure == SCHED_PRESSURE_MODEL) | |
3952 | model_start_schedule (bb); | |
3953 | } | |
7a1aca9c | 3954 | \f |
51564684 BS |
3955 | /* A structure that holds local state for the loop in schedule_block. */ |
3956 | struct sched_block_state | |
3957 | { | |
3958 | /* True if no real insns have been scheduled in the current cycle. */ | |
3959 | bool first_cycle_insn_p; | |
26965010 BS |
3960 | /* True if a shadow insn has been scheduled in the current cycle, which |
3961 | means that no more normal insns can be issued. */ | |
3962 | bool shadows_only_p; | |
06d7e8e7 BS |
3963 | /* True if we're winding down a modulo schedule, which means that we only |
3964 | issue insns with INSN_EXACT_TICK set. */ | |
3965 | bool modulo_epilogue; | |
51564684 BS |
3966 | /* Initialized with the machine's issue rate every cycle, and updated |
3967 | by calls to the variable_issue hook. */ | |
3968 | int can_issue_more; | |
3969 | }; | |
ce18efcb | 3970 | |
8c660648 | 3971 | /* INSN is the "currently executing insn". Launch each insn which was |
176f9a7b | 3972 | waiting on INSN. READY is the ready list which contains the insns |
58fb7809 VM |
3973 | that are ready to fire. CLOCK is the current cycle. The function |
3974 | returns necessary cycle advance after issuing the insn (it is not | |
3975 | zero for insns in a schedule group). */ | |
8c660648 | 3976 | |
58fb7809 | 3977 | static int |
ce1ce33a | 3978 | schedule_insn (rtx_insn *insn) |
8c660648 | 3979 | { |
e2f6ff94 MK |
3980 | sd_iterator_def sd_it; |
3981 | dep_t dep; | |
ce18efcb | 3982 | int i; |
58fb7809 | 3983 | int advance = 0; |
8c660648 | 3984 | |
fa0aee89 | 3985 | if (sched_verbose >= 1) |
8c660648 | 3986 | { |
ce18efcb | 3987 | struct reg_pressure_data *pressure_info; |
d5c67efd | 3988 | fprintf (sched_dump, ";;\t%3i--> %s %-40s:", |
7eba871a SB |
3989 | clock_var, (*current_sched_info->print_insn) (insn, 1), |
3990 | str_pattern_slim (PATTERN (insn))); | |
fae15c93 | 3991 | |
6d0de005 JH |
3992 | if (recog_memoized (insn) < 0) |
3993 | fprintf (sched_dump, "nothing"); | |
3994 | else | |
3995 | print_reservation (sched_dump, insn); | |
ce18efcb VM |
3996 | pressure_info = INSN_REG_PRESSURE (insn); |
3997 | if (pressure_info != NULL) | |
3998 | { | |
3999 | fputc (':', sched_dump); | |
1756cb66 | 4000 | for (i = 0; i < ira_pressure_classes_num; i++) |
f20f2613 VM |
4001 | fprintf (sched_dump, "%s%s%+d(%d)", |
4002 | scheduled_insns.length () > 1 | |
4003 | && INSN_LUID (insn) | |
4004 | < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "", | |
1756cb66 | 4005 | reg_class_names[ira_pressure_classes[i]], |
ce18efcb VM |
4006 | pressure_info[i].set_increase, pressure_info[i].change); |
4007 | } | |
7a1aca9c RS |
4008 | if (sched_pressure == SCHED_PRESSURE_MODEL |
4009 | && model_curr_point < model_num_insns | |
4010 | && model_index (insn) == model_curr_point) | |
4011 | fprintf (sched_dump, ":model %d", model_curr_point); | |
6d0de005 JH |
4012 | fputc ('\n', sched_dump); |
4013 | } | |
8c660648 | 4014 | |
60867e8c | 4015 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn)) |
ce18efcb VM |
4016 | update_reg_and_insn_max_reg_pressure (insn); |
4017 | ||
63f54b1a MK |
4018 | /* Scheduling instruction should have all its dependencies resolved and |
4019 | should have been removed from the ready list. */ | |
e2724e63 | 4020 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK)); |
569fa502 | 4021 | |
f49b295a | 4022 | /* Reset debug insns invalidated by moving this insn. */ |
36f52e8f | 4023 | if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn)) |
f49b295a AO |
4024 | for (sd_it = sd_iterator_start (insn, SD_LIST_BACK); |
4025 | sd_iterator_cond (&sd_it, &dep);) | |
4026 | { | |
9b517712 | 4027 | rtx_insn *dbg = DEP_PRO (dep); |
1de12eab | 4028 | struct reg_use_data *use, *next; |
f49b295a | 4029 | |
e2724e63 BS |
4030 | if (DEP_STATUS (dep) & DEP_CANCELLED) |
4031 | { | |
4032 | sd_iterator_next (&sd_it); | |
4033 | continue; | |
4034 | } | |
4035 | ||
36f52e8f | 4036 | gcc_assert (DEBUG_BIND_INSN_P (dbg)); |
f49b295a AO |
4037 | |
4038 | if (sched_verbose >= 6) | |
4039 | fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n", | |
4040 | INSN_UID (dbg)); | |
4041 | ||
4042 | /* ??? Rather than resetting the debug insn, we might be able | |
4043 | to emit a debug temp before the just-scheduled insn, but | |
4044 | this would involve checking that the expression at the | |
4045 | point of the debug insn is equivalent to the expression | |
4046 | before the just-scheduled insn. They might not be: the | |
4047 | expression in the debug insn may depend on other insns not | |
4048 | yet scheduled that set MEMs, REGs or even other debug | |
4049 | insns. It's not clear that attempting to preserve debug | |
4050 | information in these cases is worth the effort, given how | |
4051 | uncommon these resets are and the likelihood that the debug | |
4052 | temps introduced won't survive the schedule change. */ | |
4053 | INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC (); | |
4054 | df_insn_rescan (dbg); | |
4055 | ||
1de12eab JJ |
4056 | /* Unknown location doesn't use any registers. */ |
4057 | for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next) | |
4058 | { | |
14db98d4 JJ |
4059 | struct reg_use_data *prev = use; |
4060 | ||
4061 | /* Remove use from the cyclic next_regno_use chain first. */ | |
4062 | while (prev->next_regno_use != use) | |
4063 | prev = prev->next_regno_use; | |
4064 | prev->next_regno_use = use->next_regno_use; | |
1de12eab JJ |
4065 | next = use->next_insn_use; |
4066 | free (use); | |
4067 | } | |
4068 | INSN_REG_USE_LIST (dbg) = NULL; | |
4069 | ||
f49b295a AO |
4070 | /* We delete rather than resolve these deps, otherwise we |
4071 | crash in sched_free_deps(), because forward deps are | |
4072 | expected to be released before backward deps. */ | |
4073 | sd_delete_dep (sd_it); | |
4074 | } | |
4075 | ||
b198261f | 4076 | gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE); |
63f54b1a | 4077 | QUEUE_INDEX (insn) = QUEUE_SCHEDULED; |
b198261f | 4078 | |
7a1aca9c RS |
4079 | if (sched_pressure == SCHED_PRESSURE_MODEL |
4080 | && model_curr_point < model_num_insns | |
4081 | && NONDEBUG_INSN_P (insn)) | |
4082 | { | |
4083 | if (model_index (insn) == model_curr_point) | |
4084 | do | |
4085 | model_curr_point++; | |
4086 | while (model_curr_point < model_num_insns | |
4087 | && (QUEUE_INDEX (MODEL_INSN (model_curr_point)) | |
4088 | == QUEUE_SCHEDULED)); | |
4089 | else | |
4090 | model_recompute (insn); | |
4091 | model_update_limit_points (); | |
4092 | update_register_pressure (insn); | |
4093 | if (sched_verbose >= 2) | |
4094 | print_curr_reg_pressure (); | |
4095 | } | |
4096 | ||
63f54b1a MK |
4097 | gcc_assert (INSN_TICK (insn) >= MIN_TICK); |
4098 | if (INSN_TICK (insn) > clock_var) | |
4099 | /* INSN has been prematurely moved from the queue to the ready list. | |
b16abbcb BC |
4100 | This is possible only if following flags are set. */ |
4101 | gcc_assert (flag_sched_stalled_insns || sched_fusion); | |
63f54b1a MK |
4102 | |
4103 | /* ??? Probably, if INSN is scheduled prematurely, we should leave | |
4104 | INSN_TICK untouched. This is a machine-dependent issue, actually. */ | |
4105 | INSN_TICK (insn) = clock_var; | |
4106 | ||
e2724e63 BS |
4107 | check_clobbered_conditions (insn); |
4108 | ||
1a83e602 BS |
4109 | /* Update dependent instructions. First, see if by scheduling this insn |
4110 | now we broke a dependence in a way that requires us to change another | |
4111 | insn. */ | |
4112 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
4113 | sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it)) | |
4114 | { | |
4115 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
9b517712 | 4116 | rtx_insn *pro = DEP_PRO (dep); |
1a83e602 BS |
4117 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED |
4118 | && desc != NULL && desc->insn == pro) | |
4119 | apply_replacement (dep, false); | |
4120 | } | |
4121 | ||
4122 | /* Go through and resolve forward dependencies. */ | |
e2f6ff94 MK |
4123 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); |
4124 | sd_iterator_cond (&sd_it, &dep);) | |
8c660648 | 4125 | { |
9b517712 | 4126 | rtx_insn *next = DEP_CON (dep); |
e2724e63 | 4127 | bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0; |
8c660648 | 4128 | |
e2f6ff94 MK |
4129 | /* Resolve the dependence between INSN and NEXT. |
4130 | sd_resolve_dep () moves current dep to another list thus | |
4131 | advancing the iterator. */ | |
4132 | sd_resolve_dep (sd_it); | |
8c660648 | 4133 | |
e2724e63 BS |
4134 | if (cancelled) |
4135 | { | |
1a83e602 BS |
4136 | if (must_restore_pattern_p (next, dep)) |
4137 | restore_pattern (dep, false); | |
e2724e63 BS |
4138 | continue; |
4139 | } | |
4140 | ||
f49b295a AO |
4141 | /* Don't bother trying to mark next as ready if insn is a debug |
4142 | insn. If insn is the last hard dependency, it will have | |
4143 | already been discounted. */ | |
4144 | if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next)) | |
4145 | continue; | |
4146 | ||
d7bfd907 | 4147 | if (!IS_SPECULATION_BRANCHY_CHECK_P (insn)) |
496d7bb0 | 4148 | { |
b8698a0f L |
4149 | int effective_cost; |
4150 | ||
496d7bb0 | 4151 | effective_cost = try_ready (next); |
b8698a0f | 4152 | |
496d7bb0 MK |
4153 | if (effective_cost >= 0 |
4154 | && SCHED_GROUP_P (next) | |
4155 | && advance < effective_cost) | |
4156 | advance = effective_cost; | |
4157 | } | |
4158 | else | |
4159 | /* Check always has only one forward dependence (to the first insn in | |
4160 | the recovery block), therefore, this will be executed only once. */ | |
4161 | { | |
e2f6ff94 | 4162 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW)); |
496d7bb0 MK |
4163 | fix_recovery_deps (RECOVERY_BLOCK (insn)); |
4164 | } | |
8c660648 JL |
4165 | } |
4166 | ||
7a403706 | 4167 | /* Annotate the instruction with issue information -- TImode |
4bdc8810 RH |
4168 | indicates that the instruction is expected not to be able |
4169 | to issue on the same cycle as the previous insn. A machine | |
4170 | may use this information to decide how the instruction should | |
4171 | be aligned. */ | |
30028c85 | 4172 | if (issue_rate > 1 |
fae15c93 | 4173 | && GET_CODE (PATTERN (insn)) != USE |
b5b8b0ac AO |
4174 | && GET_CODE (PATTERN (insn)) != CLOBBER |
4175 | && !DEBUG_INSN_P (insn)) | |
4bdc8810 | 4176 | { |
30028c85 | 4177 | if (reload_completed) |
63f54b1a MK |
4178 | PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode); |
4179 | last_clock_var = clock_var; | |
4bdc8810 | 4180 | } |
63f54b1a | 4181 | |
06bc63d1 MK |
4182 | if (nonscheduled_insns_begin != NULL_RTX) |
4183 | /* Indicate to debug counters that INSN is scheduled. */ | |
4184 | nonscheduled_insns_begin = insn; | |
4185 | ||
58fb7809 | 4186 | return advance; |
8c660648 JL |
4187 | } |
4188 | ||
63de6c74 | 4189 | /* Functions for handling of notes. */ |
8c660648 | 4190 | |
e855c69d AB |
4191 | /* Add note list that ends on FROM_END to the end of TO_ENDP. */ |
4192 | void | |
b311fd0f | 4193 | concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp) |
e855c69d | 4194 | { |
b311fd0f | 4195 | rtx_insn *from_start; |
e855c69d | 4196 | |
e67271bd | 4197 | /* It's easy when have nothing to concat. */ |
e855c69d | 4198 | if (from_end == NULL) |
e855c69d AB |
4199 | return; |
4200 | ||
e67271bd | 4201 | /* It's also easy when destination is empty. */ |
e855c69d | 4202 | if (*to_endp == NULL) |
e855c69d AB |
4203 | { |
4204 | *to_endp = from_end; | |
4205 | return; | |
4206 | } | |
4207 | ||
4208 | from_start = from_end; | |
b8698a0f | 4209 | while (PREV_INSN (from_start) != NULL) |
e855c69d AB |
4210 | from_start = PREV_INSN (from_start); |
4211 | ||
0f82e5c9 DM |
4212 | SET_PREV_INSN (from_start) = *to_endp; |
4213 | SET_NEXT_INSN (*to_endp) = from_start; | |
e855c69d AB |
4214 | *to_endp = from_end; |
4215 | } | |
4216 | ||
e67271bd RH |
4217 | /* Delete notes between HEAD and TAIL and put them in the chain |
4218 | of notes ended by NOTE_LIST. */ | |
4219 | void | |
ce1ce33a | 4220 | remove_notes (rtx_insn *head, rtx_insn *tail) |
8c660648 | 4221 | { |
ce1ce33a | 4222 | rtx_insn *next_tail, *insn, *next; |
8c660648 | 4223 | |
e67271bd RH |
4224 | note_list = 0; |
4225 | if (head == tail && !INSN_P (head)) | |
4226 | return; | |
4227 | ||
4228 | next_tail = NEXT_INSN (tail); | |
e67271bd | 4229 | for (insn = head; insn != next_tail; insn = next) |
8c660648 | 4230 | { |
e67271bd RH |
4231 | next = NEXT_INSN (insn); |
4232 | if (!NOTE_P (insn)) | |
1124098b | 4233 | continue; |
f70b22c9 | 4234 | |
e67271bd RH |
4235 | switch (NOTE_KIND (insn)) |
4236 | { | |
4237 | case NOTE_INSN_BASIC_BLOCK: | |
e67271bd | 4238 | continue; |
8c660648 | 4239 | |
e67271bd RH |
4240 | case NOTE_INSN_EPILOGUE_BEG: |
4241 | if (insn != tail) | |
4242 | { | |
4243 | remove_insn (insn); | |
4244 | add_reg_note (next, REG_SAVE_NOTE, | |
4245 | GEN_INT (NOTE_INSN_EPILOGUE_BEG)); | |
4246 | break; | |
4247 | } | |
4248 | /* FALLTHRU */ | |
e855c69d | 4249 | |
e67271bd RH |
4250 | default: |
4251 | remove_insn (insn); | |
e855c69d | 4252 | |
e67271bd | 4253 | /* Add the note to list that ends at NOTE_LIST. */ |
0f82e5c9 DM |
4254 | SET_PREV_INSN (insn) = note_list; |
4255 | SET_NEXT_INSN (insn) = NULL_RTX; | |
e67271bd | 4256 | if (note_list) |
0f82e5c9 | 4257 | SET_NEXT_INSN (note_list) = insn; |
e67271bd RH |
4258 | note_list = insn; |
4259 | break; | |
4260 | } | |
4261 | ||
4262 | gcc_assert ((sel_sched_p () || insn != tail) && insn != head); | |
4263 | } | |
8c660648 JL |
4264 | } |
4265 | ||
26965010 BS |
4266 | /* A structure to record enough data to allow us to backtrack the scheduler to |
4267 | a previous state. */ | |
4268 | struct haifa_saved_data | |
4269 | { | |
4270 | /* Next entry on the list. */ | |
4271 | struct haifa_saved_data *next; | |
4272 | ||
4273 | /* Backtracking is associated with scheduling insns that have delay slots. | |
4274 | DELAY_PAIR points to the structure that contains the insns involved, and | |
4275 | the number of cycles between them. */ | |
4276 | struct delay_pair *delay_pair; | |
4277 | ||
4278 | /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */ | |
4279 | void *fe_saved_data; | |
4280 | /* Data used by the backend. */ | |
4281 | void *be_saved_data; | |
4282 | ||
4283 | /* Copies of global state. */ | |
4284 | int clock_var, last_clock_var; | |
4285 | struct ready_list ready; | |
4286 | state_t curr_state; | |
4287 | ||
ce1ce33a | 4288 | rtx_insn *last_scheduled_insn; |
90831096 | 4289 | rtx_insn *last_nondebug_scheduled_insn; |
ce1ce33a | 4290 | rtx_insn *nonscheduled_insns_begin; |
26965010 BS |
4291 | int cycle_issued_insns; |
4292 | ||
4293 | /* Copies of state used in the inner loop of schedule_block. */ | |
4294 | struct sched_block_state sched_block; | |
4295 | ||
4296 | /* We don't need to save q_ptr, as its value is arbitrary and we can set it | |
4297 | to 0 when restoring. */ | |
4298 | int q_size; | |
3dc99c19 | 4299 | rtx_insn_list **insn_queue; |
1a83e602 BS |
4300 | |
4301 | /* Describe pattern replacements that occurred since this backtrack point | |
4302 | was queued. */ | |
9771b263 DN |
4303 | vec<dep_t> replacement_deps; |
4304 | vec<int> replace_apply; | |
1a83e602 BS |
4305 | |
4306 | /* A copy of the next-cycle replacement vectors at the time of the backtrack | |
4307 | point. */ | |
9771b263 DN |
4308 | vec<dep_t> next_cycle_deps; |
4309 | vec<int> next_cycle_apply; | |
26965010 BS |
4310 | }; |
4311 | ||
4312 | /* A record, in reverse order, of all scheduled insns which have delay slots | |
4313 | and may require backtracking. */ | |
4314 | static struct haifa_saved_data *backtrack_queue; | |
4315 | ||
4316 | /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according | |
4317 | to SET_P. */ | |
4318 | static void | |
90831096 | 4319 | mark_backtrack_feeds (rtx_insn *insn, int set_p) |
26965010 BS |
4320 | { |
4321 | sd_iterator_def sd_it; | |
4322 | dep_t dep; | |
4323 | FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep) | |
4324 | { | |
4325 | FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p; | |
4326 | } | |
4327 | } | |
4328 | ||
26965010 BS |
4329 | /* Save the current scheduler state so that we can backtrack to it |
4330 | later if necessary. PAIR gives the insns that make it necessary to | |
4331 | save this point. SCHED_BLOCK is the local state of schedule_block | |
4332 | that need to be saved. */ | |
4333 | static void | |
4334 | save_backtrack_point (struct delay_pair *pair, | |
4335 | struct sched_block_state sched_block) | |
4336 | { | |
4337 | int i; | |
4338 | struct haifa_saved_data *save = XNEW (struct haifa_saved_data); | |
4339 | ||
4340 | save->curr_state = xmalloc (dfa_state_size); | |
4341 | memcpy (save->curr_state, curr_state, dfa_state_size); | |
4342 | ||
4343 | save->ready.first = ready.first; | |
4344 | save->ready.n_ready = ready.n_ready; | |
4345 | save->ready.n_debug = ready.n_debug; | |
4346 | save->ready.veclen = ready.veclen; | |
ce1ce33a | 4347 | save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen); |
26965010 BS |
4348 | memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx)); |
4349 | ||
3dc99c19 | 4350 | save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1); |
26965010 BS |
4351 | save->q_size = q_size; |
4352 | for (i = 0; i <= max_insn_queue_index; i++) | |
4353 | { | |
4354 | int q = NEXT_Q_AFTER (q_ptr, i); | |
e2724e63 | 4355 | save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]); |
26965010 BS |
4356 | } |
4357 | ||
4358 | save->clock_var = clock_var; | |
4359 | save->last_clock_var = last_clock_var; | |
4360 | save->cycle_issued_insns = cycle_issued_insns; | |
4361 | save->last_scheduled_insn = last_scheduled_insn; | |
4362 | save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn; | |
06bc63d1 | 4363 | save->nonscheduled_insns_begin = nonscheduled_insns_begin; |
26965010 BS |
4364 | |
4365 | save->sched_block = sched_block; | |
4366 | ||
9771b263 DN |
4367 | save->replacement_deps.create (0); |
4368 | save->replace_apply.create (0); | |
4369 | save->next_cycle_deps = next_cycle_replace_deps.copy (); | |
4370 | save->next_cycle_apply = next_cycle_apply.copy (); | |
1a83e602 | 4371 | |
26965010 BS |
4372 | if (current_sched_info->save_state) |
4373 | save->fe_saved_data = (*current_sched_info->save_state) (); | |
4374 | ||
4375 | if (targetm.sched.alloc_sched_context) | |
4376 | { | |
4377 | save->be_saved_data = targetm.sched.alloc_sched_context (); | |
4378 | targetm.sched.init_sched_context (save->be_saved_data, false); | |
4379 | } | |
4380 | else | |
4381 | save->be_saved_data = NULL; | |
4382 | ||
4383 | save->delay_pair = pair; | |
4384 | ||
4385 | save->next = backtrack_queue; | |
4386 | backtrack_queue = save; | |
4387 | ||
4388 | while (pair) | |
4389 | { | |
4390 | mark_backtrack_feeds (pair->i2, 1); | |
4391 | INSN_TICK (pair->i2) = INVALID_TICK; | |
4392 | INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair); | |
06d7e8e7 | 4393 | SHADOW_P (pair->i2) = pair->stages == 0; |
26965010 BS |
4394 | pair = pair->next_same_i1; |
4395 | } | |
4396 | } | |
4397 | ||
e2724e63 BS |
4398 | /* Walk the ready list and all queues. If any insns have unresolved backwards |
4399 | dependencies, these must be cancelled deps, broken by predication. Set or | |
4400 | clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */ | |
4401 | ||
4402 | static void | |
4403 | toggle_cancelled_flags (bool set) | |
4404 | { | |
4405 | int i; | |
4406 | sd_iterator_def sd_it; | |
4407 | dep_t dep; | |
4408 | ||
4409 | if (ready.n_ready > 0) | |
4410 | { | |
ce1ce33a | 4411 | rtx_insn **first = ready_lastpos (&ready); |
e2724e63 BS |
4412 | for (i = 0; i < ready.n_ready; i++) |
4413 | FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep) | |
4414 | if (!DEBUG_INSN_P (DEP_PRO (dep))) | |
4415 | { | |
4416 | if (set) | |
4417 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
4418 | else | |
4419 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
4420 | } | |
4421 | } | |
4422 | for (i = 0; i <= max_insn_queue_index; i++) | |
4423 | { | |
4424 | int q = NEXT_Q_AFTER (q_ptr, i); | |
96b70414 DM |
4425 | rtx_insn_list *link; |
4426 | for (link = insn_queue[q]; link; link = link->next ()) | |
e2724e63 | 4427 | { |
96b70414 | 4428 | rtx_insn *insn = link->insn (); |
e2724e63 BS |
4429 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) |
4430 | if (!DEBUG_INSN_P (DEP_PRO (dep))) | |
4431 | { | |
4432 | if (set) | |
4433 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
4434 | else | |
4435 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
4436 | } | |
4437 | } | |
4438 | } | |
4439 | } | |
4440 | ||
1a83e602 BS |
4441 | /* Undo the replacements that have occurred after backtrack point SAVE |
4442 | was placed. */ | |
4443 | static void | |
4444 | undo_replacements_for_backtrack (struct haifa_saved_data *save) | |
4445 | { | |
9771b263 | 4446 | while (!save->replacement_deps.is_empty ()) |
1a83e602 | 4447 | { |
9771b263 DN |
4448 | dep_t dep = save->replacement_deps.pop (); |
4449 | int apply_p = save->replace_apply.pop (); | |
1a83e602 BS |
4450 | |
4451 | if (apply_p) | |
4452 | restore_pattern (dep, true); | |
4453 | else | |
4454 | apply_replacement (dep, true); | |
4455 | } | |
9771b263 DN |
4456 | save->replacement_deps.release (); |
4457 | save->replace_apply.release (); | |
1a83e602 BS |
4458 | } |
4459 | ||
26965010 BS |
4460 | /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN. |
4461 | Restore their dependencies to an unresolved state, and mark them as | |
4462 | queued nowhere. */ | |
4463 | ||
4464 | static void | |
90831096 | 4465 | unschedule_insns_until (rtx_insn *insn) |
26965010 | 4466 | { |
f57aa6b0 | 4467 | auto_vec<rtx_insn *> recompute_vec; |
e2724e63 BS |
4468 | |
4469 | /* Make two passes over the insns to be unscheduled. First, we clear out | |
4470 | dependencies and other trivial bookkeeping. */ | |
26965010 BS |
4471 | for (;;) |
4472 | { | |
f57aa6b0 | 4473 | rtx_insn *last; |
26965010 BS |
4474 | sd_iterator_def sd_it; |
4475 | dep_t dep; | |
4476 | ||
9771b263 | 4477 | last = scheduled_insns.pop (); |
26965010 BS |
4478 | |
4479 | /* This will be changed by restore_backtrack_point if the insn is in | |
4480 | any queue. */ | |
4481 | QUEUE_INDEX (last) = QUEUE_NOWHERE; | |
4482 | if (last != insn) | |
4483 | INSN_TICK (last) = INVALID_TICK; | |
4484 | ||
06d7e8e7 BS |
4485 | if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid) |
4486 | modulo_insns_scheduled--; | |
4487 | ||
26965010 BS |
4488 | for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW); |
4489 | sd_iterator_cond (&sd_it, &dep);) | |
4490 | { | |
9b517712 | 4491 | rtx_insn *con = DEP_CON (dep); |
26965010 | 4492 | sd_unresolve_dep (sd_it); |
e2724e63 BS |
4493 | if (!MUST_RECOMPUTE_SPEC_P (con)) |
4494 | { | |
4495 | MUST_RECOMPUTE_SPEC_P (con) = 1; | |
9771b263 | 4496 | recompute_vec.safe_push (con); |
e2724e63 | 4497 | } |
26965010 BS |
4498 | } |
4499 | ||
4500 | if (last == insn) | |
4501 | break; | |
4502 | } | |
e2724e63 BS |
4503 | |
4504 | /* A second pass, to update ready and speculation status for insns | |
4505 | depending on the unscheduled ones. The first pass must have | |
4506 | popped the scheduled_insns vector up to the point where we | |
4507 | restart scheduling, as recompute_todo_spec requires it to be | |
4508 | up-to-date. */ | |
9771b263 | 4509 | while (!recompute_vec.is_empty ()) |
e2724e63 | 4510 | { |
f57aa6b0 | 4511 | rtx_insn *con; |
e2724e63 | 4512 | |
9771b263 | 4513 | con = recompute_vec.pop (); |
e2724e63 BS |
4514 | MUST_RECOMPUTE_SPEC_P (con) = 0; |
4515 | if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK)) | |
4516 | { | |
4517 | TODO_SPEC (con) = HARD_DEP; | |
4518 | INSN_TICK (con) = INVALID_TICK; | |
4519 | if (PREDICATED_PAT (con) != NULL_RTX) | |
4520 | haifa_change_pattern (con, ORIG_PAT (con)); | |
4521 | } | |
4522 | else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED) | |
1a83e602 | 4523 | TODO_SPEC (con) = recompute_todo_spec (con, true); |
e2724e63 | 4524 | } |
26965010 BS |
4525 | } |
4526 | ||
4527 | /* Restore scheduler state from the topmost entry on the backtracking queue. | |
4528 | PSCHED_BLOCK_P points to the local data of schedule_block that we must | |
4529 | overwrite with the saved data. | |
4530 | The caller must already have called unschedule_insns_until. */ | |
4531 | ||
4532 | static void | |
4533 | restore_last_backtrack_point (struct sched_block_state *psched_block) | |
26965010 | 4534 | { |
26965010 BS |
4535 | int i; |
4536 | struct haifa_saved_data *save = backtrack_queue; | |
4537 | ||
4538 | backtrack_queue = save->next; | |
4539 | ||
4540 | if (current_sched_info->restore_state) | |
4541 | (*current_sched_info->restore_state) (save->fe_saved_data); | |
4542 | ||
4543 | if (targetm.sched.alloc_sched_context) | |
4544 | { | |
4545 | targetm.sched.set_sched_context (save->be_saved_data); | |
4546 | targetm.sched.free_sched_context (save->be_saved_data); | |
4547 | } | |
4548 | ||
1a83e602 BS |
4549 | /* Do this first since it clobbers INSN_TICK of the involved |
4550 | instructions. */ | |
4551 | undo_replacements_for_backtrack (save); | |
4552 | ||
26965010 BS |
4553 | /* Clear the QUEUE_INDEX of everything in the ready list or one |
4554 | of the queues. */ | |
4555 | if (ready.n_ready > 0) | |
4556 | { | |
ce1ce33a | 4557 | rtx_insn **first = ready_lastpos (&ready); |
26965010 BS |
4558 | for (i = 0; i < ready.n_ready; i++) |
4559 | { | |
ce1ce33a | 4560 | rtx_insn *insn = first[i]; |
e2724e63 BS |
4561 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; |
4562 | INSN_TICK (insn) = INVALID_TICK; | |
26965010 BS |
4563 | } |
4564 | } | |
4565 | for (i = 0; i <= max_insn_queue_index; i++) | |
4566 | { | |
4567 | int q = NEXT_Q_AFTER (q_ptr, i); | |
4568 | ||
96b70414 | 4569 | for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ()) |
26965010 | 4570 | { |
96b70414 | 4571 | rtx_insn *x = link->insn (); |
26965010 BS |
4572 | QUEUE_INDEX (x) = QUEUE_NOWHERE; |
4573 | INSN_TICK (x) = INVALID_TICK; | |
4574 | } | |
4575 | free_INSN_LIST_list (&insn_queue[q]); | |
4576 | } | |
4577 | ||
4578 | free (ready.vec); | |
4579 | ready = save->ready; | |
4580 | ||
4581 | if (ready.n_ready > 0) | |
4582 | { | |
ce1ce33a | 4583 | rtx_insn **first = ready_lastpos (&ready); |
26965010 BS |
4584 | for (i = 0; i < ready.n_ready; i++) |
4585 | { | |
ce1ce33a | 4586 | rtx_insn *insn = first[i]; |
e2724e63 | 4587 | QUEUE_INDEX (insn) = QUEUE_READY; |
1a83e602 | 4588 | TODO_SPEC (insn) = recompute_todo_spec (insn, true); |
e2724e63 | 4589 | INSN_TICK (insn) = save->clock_var; |
26965010 BS |
4590 | } |
4591 | } | |
4592 | ||
4593 | q_ptr = 0; | |
4594 | q_size = save->q_size; | |
4595 | for (i = 0; i <= max_insn_queue_index; i++) | |
4596 | { | |
4597 | int q = NEXT_Q_AFTER (q_ptr, i); | |
4598 | ||
4599 | insn_queue[q] = save->insn_queue[q]; | |
4600 | ||
96b70414 | 4601 | for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ()) |
26965010 | 4602 | { |
96b70414 | 4603 | rtx_insn *x = link->insn (); |
26965010 | 4604 | QUEUE_INDEX (x) = i; |
1a83e602 | 4605 | TODO_SPEC (x) = recompute_todo_spec (x, true); |
26965010 BS |
4606 | INSN_TICK (x) = save->clock_var + i; |
4607 | } | |
4608 | } | |
4609 | free (save->insn_queue); | |
4610 | ||
e2724e63 BS |
4611 | toggle_cancelled_flags (true); |
4612 | ||
26965010 BS |
4613 | clock_var = save->clock_var; |
4614 | last_clock_var = save->last_clock_var; | |
4615 | cycle_issued_insns = save->cycle_issued_insns; | |
4616 | last_scheduled_insn = save->last_scheduled_insn; | |
4617 | last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn; | |
06bc63d1 | 4618 | nonscheduled_insns_begin = save->nonscheduled_insns_begin; |
26965010 BS |
4619 | |
4620 | *psched_block = save->sched_block; | |
4621 | ||
4622 | memcpy (curr_state, save->curr_state, dfa_state_size); | |
4623 | free (save->curr_state); | |
4624 | ||
4625 | mark_backtrack_feeds (save->delay_pair->i2, 0); | |
4626 | ||
9771b263 DN |
4627 | gcc_assert (next_cycle_replace_deps.is_empty ()); |
4628 | next_cycle_replace_deps = save->next_cycle_deps.copy (); | |
4629 | next_cycle_apply = save->next_cycle_apply.copy (); | |
1a83e602 | 4630 | |
26965010 BS |
4631 | free (save); |
4632 | ||
4633 | for (save = backtrack_queue; save; save = save->next) | |
4634 | { | |
4635 | mark_backtrack_feeds (save->delay_pair->i2, 1); | |
4636 | } | |
4637 | } | |
4638 | ||
4639 | /* Discard all data associated with the topmost entry in the backtrack | |
4640 | queue. If RESET_TICK is false, we just want to free the data. If true, | |
4641 | we are doing this because we discovered a reason to backtrack. In the | |
4642 | latter case, also reset the INSN_TICK for the shadow insn. */ | |
4643 | static void | |
4644 | free_topmost_backtrack_point (bool reset_tick) | |
4645 | { | |
4646 | struct haifa_saved_data *save = backtrack_queue; | |
4647 | int i; | |
4648 | ||
4649 | backtrack_queue = save->next; | |
4650 | ||
4651 | if (reset_tick) | |
4652 | { | |
4653 | struct delay_pair *pair = save->delay_pair; | |
4654 | while (pair) | |
4655 | { | |
4656 | INSN_TICK (pair->i2) = INVALID_TICK; | |
4657 | INSN_EXACT_TICK (pair->i2) = INVALID_TICK; | |
4658 | pair = pair->next_same_i1; | |
4659 | } | |
1a83e602 BS |
4660 | undo_replacements_for_backtrack (save); |
4661 | } | |
4662 | else | |
4663 | { | |
9771b263 DN |
4664 | save->replacement_deps.release (); |
4665 | save->replace_apply.release (); | |
26965010 | 4666 | } |
1a83e602 | 4667 | |
26965010 BS |
4668 | if (targetm.sched.free_sched_context) |
4669 | targetm.sched.free_sched_context (save->be_saved_data); | |
4670 | if (current_sched_info->restore_state) | |
4671 | free (save->fe_saved_data); | |
4672 | for (i = 0; i <= max_insn_queue_index; i++) | |
4673 | free_INSN_LIST_list (&save->insn_queue[i]); | |
4674 | free (save->insn_queue); | |
4675 | free (save->curr_state); | |
4676 | free (save->ready.vec); | |
4677 | free (save); | |
4678 | } | |
4679 | ||
4680 | /* Free the entire backtrack queue. */ | |
4681 | static void | |
4682 | free_backtrack_queue (void) | |
4683 | { | |
4684 | while (backtrack_queue) | |
4685 | free_topmost_backtrack_point (false); | |
4686 | } | |
4687 | ||
1a83e602 BS |
4688 | /* Apply a replacement described by DESC. If IMMEDIATELY is false, we |
4689 | may have to postpone the replacement until the start of the next cycle, | |
4690 | at which point we will be called again with IMMEDIATELY true. This is | |
4691 | only done for machines which have instruction packets with explicit | |
4692 | parallelism however. */ | |
4693 | static void | |
4694 | apply_replacement (dep_t dep, bool immediately) | |
4695 | { | |
4696 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
4697 | if (!immediately && targetm.sched.exposed_pipeline && reload_completed) | |
4698 | { | |
9771b263 DN |
4699 | next_cycle_replace_deps.safe_push (dep); |
4700 | next_cycle_apply.safe_push (1); | |
1a83e602 BS |
4701 | } |
4702 | else | |
4703 | { | |
4704 | bool success; | |
4705 | ||
4706 | if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED) | |
4707 | return; | |
4708 | ||
4709 | if (sched_verbose >= 5) | |
4710 | fprintf (sched_dump, "applying replacement for insn %d\n", | |
4711 | INSN_UID (desc->insn)); | |
4712 | ||
4713 | success = validate_change (desc->insn, desc->loc, desc->newval, 0); | |
4714 | gcc_assert (success); | |
4715 | ||
079c81d0 RD |
4716 | rtx_insn *insn = DEP_PRO (dep); |
4717 | ||
4718 | /* Recompute priority since dependent priorities may have changed. */ | |
4719 | priority (insn, true); | |
1a83e602 | 4720 | update_insn_after_change (desc->insn); |
079c81d0 | 4721 | |
1a83e602 BS |
4722 | if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0) |
4723 | fix_tick_ready (desc->insn); | |
4724 | ||
4725 | if (backtrack_queue != NULL) | |
4726 | { | |
9771b263 DN |
4727 | backtrack_queue->replacement_deps.safe_push (dep); |
4728 | backtrack_queue->replace_apply.safe_push (1); | |
1a83e602 BS |
4729 | } |
4730 | } | |
4731 | } | |
4732 | ||
4733 | /* We have determined that a pattern involved in DEP must be restored. | |
4734 | If IMMEDIATELY is false, we may have to postpone the replacement | |
4735 | until the start of the next cycle, at which point we will be called | |
4736 | again with IMMEDIATELY true. */ | |
4737 | static void | |
4738 | restore_pattern (dep_t dep, bool immediately) | |
4739 | { | |
9b517712 | 4740 | rtx_insn *next = DEP_CON (dep); |
1a83e602 BS |
4741 | int tick = INSN_TICK (next); |
4742 | ||
4743 | /* If we already scheduled the insn, the modified version is | |
4744 | correct. */ | |
4745 | if (QUEUE_INDEX (next) == QUEUE_SCHEDULED) | |
4746 | return; | |
4747 | ||
4748 | if (!immediately && targetm.sched.exposed_pipeline && reload_completed) | |
4749 | { | |
9771b263 DN |
4750 | next_cycle_replace_deps.safe_push (dep); |
4751 | next_cycle_apply.safe_push (0); | |
1a83e602 BS |
4752 | return; |
4753 | } | |
4754 | ||
4755 | ||
4756 | if (DEP_TYPE (dep) == REG_DEP_CONTROL) | |
4757 | { | |
4758 | if (sched_verbose >= 5) | |
4759 | fprintf (sched_dump, "restoring pattern for insn %d\n", | |
4760 | INSN_UID (next)); | |
4761 | haifa_change_pattern (next, ORIG_PAT (next)); | |
4762 | } | |
4763 | else | |
4764 | { | |
4765 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
4766 | bool success; | |
4767 | ||
4768 | if (sched_verbose >= 5) | |
4769 | fprintf (sched_dump, "restoring pattern for insn %d\n", | |
4770 | INSN_UID (desc->insn)); | |
4771 | tick = INSN_TICK (desc->insn); | |
4772 | ||
4773 | success = validate_change (desc->insn, desc->loc, desc->orig, 0); | |
4774 | gcc_assert (success); | |
079c81d0 RD |
4775 | |
4776 | rtx_insn *insn = DEP_PRO (dep); | |
4777 | ||
4778 | if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED) | |
4779 | { | |
4780 | /* Recompute priority since dependent priorities may have changed. */ | |
4781 | priority (insn, true); | |
4782 | } | |
4783 | ||
1a83e602 | 4784 | update_insn_after_change (desc->insn); |
079c81d0 | 4785 | |
1a83e602 BS |
4786 | if (backtrack_queue != NULL) |
4787 | { | |
9771b263 DN |
4788 | backtrack_queue->replacement_deps.safe_push (dep); |
4789 | backtrack_queue->replace_apply.safe_push (0); | |
1a83e602 BS |
4790 | } |
4791 | } | |
4792 | INSN_TICK (next) = tick; | |
4793 | if (TODO_SPEC (next) == DEP_POSTPONED) | |
4794 | return; | |
4795 | ||
4796 | if (sd_lists_empty_p (next, SD_LIST_BACK)) | |
4797 | TODO_SPEC (next) = 0; | |
4798 | else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK)) | |
4799 | TODO_SPEC (next) = HARD_DEP; | |
4800 | } | |
4801 | ||
4802 | /* Perform pattern replacements that were queued up until the next | |
4803 | cycle. */ | |
4804 | static void | |
4805 | perform_replacements_new_cycle (void) | |
4806 | { | |
4807 | int i; | |
4808 | dep_t dep; | |
9771b263 | 4809 | FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep) |
1a83e602 | 4810 | { |
9771b263 | 4811 | int apply_p = next_cycle_apply[i]; |
1a83e602 BS |
4812 | if (apply_p) |
4813 | apply_replacement (dep, true); | |
4814 | else | |
4815 | restore_pattern (dep, true); | |
4816 | } | |
9771b263 DN |
4817 | next_cycle_replace_deps.truncate (0); |
4818 | next_cycle_apply.truncate (0); | |
1a83e602 BS |
4819 | } |
4820 | ||
26965010 BS |
4821 | /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of |
4822 | instructions we've previously encountered, a set bit prevents | |
4823 | recursion. BUDGET is a limit on how far ahead we look, it is | |
4824 | reduced on recursive calls. Return true if we produced a good | |
4825 | estimate, or false if we exceeded the budget. */ | |
4826 | static bool | |
f57aa6b0 | 4827 | estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget) |
26965010 BS |
4828 | { |
4829 | sd_iterator_def sd_it; | |
4830 | dep_t dep; | |
4831 | int earliest = INSN_TICK (insn); | |
4832 | ||
4833 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
4834 | { | |
9b517712 | 4835 | rtx_insn *pro = DEP_PRO (dep); |
26965010 BS |
4836 | int t; |
4837 | ||
e2724e63 BS |
4838 | if (DEP_STATUS (dep) & DEP_CANCELLED) |
4839 | continue; | |
4840 | ||
26965010 BS |
4841 | if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED) |
4842 | gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn)); | |
4843 | else | |
4844 | { | |
4845 | int cost = dep_cost (dep); | |
4846 | if (cost >= budget) | |
4847 | return false; | |
4848 | if (!bitmap_bit_p (processed, INSN_LUID (pro))) | |
4849 | { | |
4850 | if (!estimate_insn_tick (processed, pro, budget - cost)) | |
4851 | return false; | |
4852 | } | |
4853 | gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK); | |
4854 | t = INSN_TICK_ESTIMATE (pro) + cost; | |
4855 | if (earliest == INVALID_TICK || t > earliest) | |
4856 | earliest = t; | |
4857 | } | |
4858 | } | |
4859 | bitmap_set_bit (processed, INSN_LUID (insn)); | |
4860 | INSN_TICK_ESTIMATE (insn) = earliest; | |
4861 | return true; | |
4862 | } | |
4863 | ||
4864 | /* Examine the pair of insns in P, and estimate (optimistically, assuming | |
4865 | infinite resources) the cycle in which the delayed shadow can be issued. | |
4866 | Return the number of cycles that must pass before the real insn can be | |
4867 | issued in order to meet this constraint. */ | |
4868 | static int | |
4869 | estimate_shadow_tick (struct delay_pair *p) | |
4870 | { | |
8f9b31f7 | 4871 | auto_bitmap processed; |
26965010 BS |
4872 | int t; |
4873 | bool cutoff; | |
26965010 | 4874 | |
8f9b31f7 | 4875 | cutoff = !estimate_insn_tick (processed, p->i2, |
26965010 | 4876 | max_insn_queue_index + pair_delay (p)); |
26965010 BS |
4877 | if (cutoff) |
4878 | return max_insn_queue_index; | |
4879 | t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1); | |
4880 | if (t > 0) | |
4881 | return t; | |
4882 | return 0; | |
4883 | } | |
e67271bd | 4884 | |
06d7e8e7 BS |
4885 | /* If INSN has no unresolved backwards dependencies, add it to the schedule and |
4886 | recursively resolve all its forward dependencies. */ | |
4887 | static void | |
ce1ce33a | 4888 | resolve_dependencies (rtx_insn *insn) |
06d7e8e7 BS |
4889 | { |
4890 | sd_iterator_def sd_it; | |
4891 | dep_t dep; | |
4892 | ||
4893 | /* Don't use sd_lists_empty_p; it ignores debug insns. */ | |
4894 | if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL | |
4895 | || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL) | |
4896 | return; | |
4897 | ||
4898 | if (sched_verbose >= 4) | |
4899 | fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn)); | |
4900 | ||
4901 | if (QUEUE_INDEX (insn) >= 0) | |
4902 | queue_remove (insn); | |
4903 | ||
9771b263 | 4904 | scheduled_insns.safe_push (insn); |
06d7e8e7 BS |
4905 | |
4906 | /* Update dependent instructions. */ | |
4907 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); | |
4908 | sd_iterator_cond (&sd_it, &dep);) | |
4909 | { | |
9b517712 | 4910 | rtx_insn *next = DEP_CON (dep); |
06d7e8e7 BS |
4911 | |
4912 | if (sched_verbose >= 4) | |
4913 | fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn), | |
4914 | INSN_UID (next)); | |
4915 | ||
4916 | /* Resolve the dependence between INSN and NEXT. | |
4917 | sd_resolve_dep () moves current dep to another list thus | |
4918 | advancing the iterator. */ | |
4919 | sd_resolve_dep (sd_it); | |
4920 | ||
4921 | if (!IS_SPECULATION_BRANCHY_CHECK_P (insn)) | |
4922 | { | |
4923 | resolve_dependencies (next); | |
4924 | } | |
4925 | else | |
4926 | /* Check always has only one forward dependence (to the first insn in | |
4927 | the recovery block), therefore, this will be executed only once. */ | |
4928 | { | |
4929 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW)); | |
4930 | } | |
4931 | } | |
4932 | } | |
4933 | ||
4934 | ||
496d7bb0 MK |
4935 | /* Return the head and tail pointers of ebb starting at BEG and ending |
4936 | at END. */ | |
b4ead7d4 | 4937 | void |
52d251b5 DM |
4938 | get_ebb_head_tail (basic_block beg, basic_block end, |
4939 | rtx_insn **headp, rtx_insn **tailp) | |
496d7bb0 | 4940 | { |
52d251b5 DM |
4941 | rtx_insn *beg_head = BB_HEAD (beg); |
4942 | rtx_insn * beg_tail = BB_END (beg); | |
4943 | rtx_insn * end_head = BB_HEAD (end); | |
4944 | rtx_insn * end_tail = BB_END (end); | |
496d7bb0 MK |
4945 | |
4946 | /* Don't include any notes or labels at the beginning of the BEG | |
4947 | basic block, or notes at the end of the END basic blocks. */ | |
4948 | ||
4949 | if (LABEL_P (beg_head)) | |
4950 | beg_head = NEXT_INSN (beg_head); | |
4951 | ||
4952 | while (beg_head != beg_tail) | |
a59d15cf | 4953 | if (NOTE_P (beg_head)) |
496d7bb0 | 4954 | beg_head = NEXT_INSN (beg_head); |
a59d15cf AO |
4955 | else if (DEBUG_INSN_P (beg_head)) |
4956 | { | |
52d251b5 | 4957 | rtx_insn * note, *next; |
a59d15cf AO |
4958 | |
4959 | for (note = NEXT_INSN (beg_head); | |
4960 | note != beg_tail; | |
4961 | note = next) | |
4962 | { | |
4963 | next = NEXT_INSN (note); | |
4964 | if (NOTE_P (note)) | |
4965 | { | |
4966 | if (sched_verbose >= 9) | |
4967 | fprintf (sched_dump, "reorder %i\n", INSN_UID (note)); | |
4968 | ||
4969 | reorder_insns_nobb (note, note, PREV_INSN (beg_head)); | |
4970 | ||
4971 | if (BLOCK_FOR_INSN (note) != beg) | |
4972 | df_insn_change_bb (note, beg); | |
4973 | } | |
4974 | else if (!DEBUG_INSN_P (note)) | |
4975 | break; | |
4976 | } | |
4977 | ||
4978 | break; | |
4979 | } | |
496d7bb0 MK |
4980 | else |
4981 | break; | |
4982 | ||
4983 | *headp = beg_head; | |
4984 | ||
4985 | if (beg == end) | |
4986 | end_head = beg_head; | |
4987 | else if (LABEL_P (end_head)) | |
4988 | end_head = NEXT_INSN (end_head); | |
4989 | ||
4990 | while (end_head != end_tail) | |
a59d15cf | 4991 | if (NOTE_P (end_tail)) |
496d7bb0 | 4992 | end_tail = PREV_INSN (end_tail); |
a59d15cf AO |
4993 | else if (DEBUG_INSN_P (end_tail)) |
4994 | { | |
52d251b5 | 4995 | rtx_insn * note, *prev; |
a59d15cf AO |
4996 | |
4997 | for (note = PREV_INSN (end_tail); | |
4998 | note != end_head; | |
4999 | note = prev) | |
5000 | { | |
5001 | prev = PREV_INSN (note); | |
5002 | if (NOTE_P (note)) | |
5003 | { | |
5004 | if (sched_verbose >= 9) | |
5005 | fprintf (sched_dump, "reorder %i\n", INSN_UID (note)); | |
5006 | ||
5007 | reorder_insns_nobb (note, note, end_tail); | |
5008 | ||
5009 | if (end_tail == BB_END (end)) | |
1130d5e3 | 5010 | BB_END (end) = note; |
a59d15cf AO |
5011 | |
5012 | if (BLOCK_FOR_INSN (note) != end) | |
5013 | df_insn_change_bb (note, end); | |
5014 | } | |
5015 | else if (!DEBUG_INSN_P (note)) | |
5016 | break; | |
5017 | } | |
5018 | ||
5019 | break; | |
5020 | } | |
496d7bb0 MK |
5021 | else |
5022 | break; | |
8c660648 | 5023 | |
496d7bb0 | 5024 | *tailp = end_tail; |
8c660648 JL |
5025 | } |
5026 | ||
1708fd40 BS |
5027 | /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */ |
5028 | ||
b4ead7d4 | 5029 | int |
dc01c3d1 | 5030 | no_real_insns_p (const rtx_insn *head, const rtx_insn *tail) |
1708fd40 BS |
5031 | { |
5032 | while (head != NEXT_INSN (tail)) | |
5033 | { | |
a59d15cf | 5034 | if (!NOTE_P (head) && !LABEL_P (head)) |
1708fd40 BS |
5035 | return 0; |
5036 | head = NEXT_INSN (head); | |
5037 | } | |
5038 | return 1; | |
5039 | } | |
5040 | ||
e855c69d AB |
5041 | /* Restore-other-notes: NOTE_LIST is the end of a chain of notes |
5042 | previously found among the insns. Insert them just before HEAD. */ | |
ce1ce33a DM |
5043 | rtx_insn * |
5044 | restore_other_notes (rtx_insn *head, basic_block head_bb) | |
e855c69d AB |
5045 | { |
5046 | if (note_list != 0) | |
5047 | { | |
ce1ce33a | 5048 | rtx_insn *note_head = note_list; |
e855c69d AB |
5049 | |
5050 | if (head) | |
5051 | head_bb = BLOCK_FOR_INSN (head); | |
5052 | else | |
5053 | head = NEXT_INSN (bb_note (head_bb)); | |
5054 | ||
5055 | while (PREV_INSN (note_head)) | |
5056 | { | |
5057 | set_block_for_insn (note_head, head_bb); | |
5058 | note_head = PREV_INSN (note_head); | |
8c660648 | 5059 | } |
e855c69d AB |
5060 | /* In the above cycle we've missed this note. */ |
5061 | set_block_for_insn (note_head, head_bb); | |
5062 | ||
0f82e5c9 DM |
5063 | SET_PREV_INSN (note_head) = PREV_INSN (head); |
5064 | SET_NEXT_INSN (PREV_INSN (head)) = note_head; | |
5065 | SET_PREV_INSN (head) = note_list; | |
5066 | SET_NEXT_INSN (note_list) = head; | |
e855c69d AB |
5067 | |
5068 | if (BLOCK_FOR_INSN (head) != head_bb) | |
1130d5e3 | 5069 | BB_END (head_bb) = note_list; |
e855c69d AB |
5070 | |
5071 | head = note_head; | |
8c660648 | 5072 | } |
e855c69d AB |
5073 | |
5074 | return head; | |
8c660648 JL |
5075 | } |
5076 | ||
1a83e602 BS |
5077 | /* When we know we are going to discard the schedule due to a failed attempt |
5078 | at modulo scheduling, undo all replacements. */ | |
5079 | static void | |
5080 | undo_all_replacements (void) | |
5081 | { | |
ce1ce33a | 5082 | rtx_insn *insn; |
1a83e602 BS |
5083 | int i; |
5084 | ||
9771b263 | 5085 | FOR_EACH_VEC_ELT (scheduled_insns, i, insn) |
1a83e602 BS |
5086 | { |
5087 | sd_iterator_def sd_it; | |
5088 | dep_t dep; | |
5089 | ||
5090 | /* See if we must undo a replacement. */ | |
5091 | for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW); | |
5092 | sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it)) | |
5093 | { | |
5094 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
5095 | if (desc != NULL) | |
5096 | validate_change (desc->insn, desc->loc, desc->orig, 0); | |
5097 | } | |
5098 | } | |
5099 | } | |
5100 | ||
06bc63d1 MK |
5101 | /* Return first non-scheduled insn in the current scheduling block. |
5102 | This is mostly used for debug-counter purposes. */ | |
ce1ce33a | 5103 | static rtx_insn * |
06bc63d1 MK |
5104 | first_nonscheduled_insn (void) |
5105 | { | |
ce1ce33a DM |
5106 | rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX |
5107 | ? nonscheduled_insns_begin | |
5108 | : current_sched_info->prev_head); | |
06bc63d1 MK |
5109 | |
5110 | do | |
5111 | { | |
5112 | insn = next_nonnote_nondebug_insn (insn); | |
5113 | } | |
5114 | while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED); | |
5115 | ||
5116 | return insn; | |
5117 | } | |
5118 | ||
8c660648 JL |
5119 | /* Move insns that became ready to fire from queue to ready list. */ |
5120 | ||
176f9a7b | 5121 | static void |
1d088dee | 5122 | queue_to_ready (struct ready_list *ready) |
8c660648 | 5123 | { |
ce1ce33a | 5124 | rtx_insn *insn; |
3dc99c19 | 5125 | rtx_insn_list *link; |
90831096 | 5126 | rtx_insn *skip_insn; |
8c660648 JL |
5127 | |
5128 | q_ptr = NEXT_Q (q_ptr); | |
5129 | ||
b631c5f7 | 5130 | if (dbg_cnt (sched_insn) == false) |
06bc63d1 MK |
5131 | /* If debug counter is activated do not requeue the first |
5132 | nonscheduled insn. */ | |
5133 | skip_insn = first_nonscheduled_insn (); | |
b631c5f7 | 5134 | else |
90831096 | 5135 | skip_insn = NULL; |
b631c5f7 | 5136 | |
8c660648 | 5137 | /* Add all pending insns that can be scheduled without stalls to the |
b4ead7d4 | 5138 | ready list. */ |
3dc99c19 | 5139 | for (link = insn_queue[q_ptr]; link; link = link->next ()) |
b4ead7d4 | 5140 | { |
3dc99c19 | 5141 | insn = link->insn (); |
b4ead7d4 | 5142 | q_size -= 1; |
1708fd40 | 5143 | |
b4ead7d4 BS |
5144 | if (sched_verbose >= 2) |
5145 | fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ", | |
5146 | (*current_sched_info->print_insn) (insn, 0)); | |
1708fd40 | 5147 | |
6f8dd94b EB |
5148 | /* If the ready list is full, delay the insn for 1 cycle. |
5149 | See the comment in schedule_block for the rationale. */ | |
5150 | if (!reload_completed | |
7a1aca9c RS |
5151 | && (ready->n_ready - ready->n_debug > MAX_SCHED_READY_INSNS |
5152 | || (sched_pressure == SCHED_PRESSURE_MODEL | |
5153 | /* Limit pressure recalculations to MAX_SCHED_READY_INSNS | |
5154 | instructions too. */ | |
5155 | && model_index (insn) > (model_curr_point | |
5156 | + MAX_SCHED_READY_INSNS))) | |
5157 | && !(sched_pressure == SCHED_PRESSURE_MODEL | |
5158 | && model_curr_point < model_num_insns | |
5159 | /* Always allow the next model instruction to issue. */ | |
5160 | && model_index (insn) == model_curr_point) | |
b631c5f7 SP |
5161 | && !SCHED_GROUP_P (insn) |
5162 | && insn != skip_insn) | |
d5c67efd MK |
5163 | { |
5164 | if (sched_verbose >= 2) | |
5165 | fprintf (sched_dump, "keeping in queue, ready full\n"); | |
5166 | queue_insn (insn, 1, "ready full"); | |
5167 | } | |
6f8dd94b EB |
5168 | else |
5169 | { | |
5170 | ready_add (ready, insn, false); | |
5171 | if (sched_verbose >= 2) | |
5172 | fprintf (sched_dump, "moving to ready without stalls\n"); | |
5173 | } | |
1708fd40 | 5174 | } |
63f54b1a | 5175 | free_INSN_LIST_list (&insn_queue[q_ptr]); |
b4ead7d4 BS |
5176 | |
5177 | /* If there are no ready insns, stall until one is ready and add all | |
5178 | of the pending insns at that point to the ready list. */ | |
5179 | if (ready->n_ready == 0) | |
1708fd40 | 5180 | { |
b3694847 | 5181 | int stalls; |
1708fd40 | 5182 | |
fa0aee89 | 5183 | for (stalls = 1; stalls <= max_insn_queue_index; stalls++) |
b4ead7d4 BS |
5184 | { |
5185 | if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)])) | |
5186 | { | |
3dc99c19 | 5187 | for (; link; link = link->next ()) |
b4ead7d4 | 5188 | { |
96b70414 | 5189 | insn = link->insn (); |
b4ead7d4 | 5190 | q_size -= 1; |
1708fd40 | 5191 | |
b4ead7d4 BS |
5192 | if (sched_verbose >= 2) |
5193 | fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ", | |
5194 | (*current_sched_info->print_insn) (insn, 0)); | |
1708fd40 | 5195 | |
63f54b1a | 5196 | ready_add (ready, insn, false); |
b4ead7d4 BS |
5197 | if (sched_verbose >= 2) |
5198 | fprintf (sched_dump, "moving to ready with %d stalls\n", stalls); | |
5199 | } | |
63f54b1a | 5200 | free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]); |
1708fd40 | 5201 | |
fae15c93 VM |
5202 | advance_one_cycle (); |
5203 | ||
5204 | break; | |
b4ead7d4 | 5205 | } |
fae15c93 VM |
5206 | |
5207 | advance_one_cycle (); | |
b4ead7d4 | 5208 | } |
1708fd40 | 5209 | |
b4ead7d4 BS |
5210 | q_ptr = NEXT_Q_AFTER (q_ptr, stalls); |
5211 | clock_var += stalls; | |
d5c67efd MK |
5212 | if (sched_verbose >= 2) |
5213 | fprintf (sched_dump, ";;\tAdvancing clock by %d cycle[s] to %d\n", | |
5214 | stalls, clock_var); | |
1708fd40 | 5215 | } |
1708fd40 BS |
5216 | } |
5217 | ||
569fa502 | 5218 | /* Used by early_queue_to_ready. Determines whether it is "ok" to |
b8698a0f L |
5219 | prematurely move INSN from the queue to the ready list. Currently, |
5220 | if a target defines the hook 'is_costly_dependence', this function | |
569fa502 | 5221 | uses the hook to check whether there exist any dependences which are |
b8698a0f | 5222 | considered costly by the target, between INSN and other insns that |
569fa502 DN |
5223 | have already been scheduled. Dependences are checked up to Y cycles |
5224 | back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows | |
b8698a0f L |
5225 | controlling this value. |
5226 | (Other considerations could be taken into account instead (or in | |
569fa502 DN |
5227 | addition) depending on user flags and target hooks. */ |
5228 | ||
b8698a0f | 5229 | static bool |
90831096 | 5230 | ok_for_early_queue_removal (rtx_insn *insn) |
569fa502 | 5231 | { |
569fa502 DN |
5232 | if (targetm.sched.is_costly_dependence) |
5233 | { | |
9b69cf83 | 5234 | int n_cycles; |
9771b263 | 5235 | int i = scheduled_insns.length (); |
569fa502 DN |
5236 | for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--) |
5237 | { | |
9b69cf83 | 5238 | while (i-- > 0) |
569fa502 | 5239 | { |
b198261f | 5240 | int cost; |
569fa502 | 5241 | |
21afc57d | 5242 | rtx_insn *prev_insn = scheduled_insns[i]; |
ec8628e8 | 5243 | |
4b4bf941 | 5244 | if (!NOTE_P (prev_insn)) |
569fa502 | 5245 | { |
e2f6ff94 | 5246 | dep_t dep; |
b198261f | 5247 | |
e2f6ff94 | 5248 | dep = sd_find_dep_between (prev_insn, insn, true); |
b198261f | 5249 | |
e2f6ff94 | 5250 | if (dep != NULL) |
569fa502 | 5251 | { |
b198261f MK |
5252 | cost = dep_cost (dep); |
5253 | ||
5254 | if (targetm.sched.is_costly_dependence (dep, cost, | |
569fa502 DN |
5255 | flag_sched_stalled_insns_dep - n_cycles)) |
5256 | return false; | |
5257 | } | |
5258 | } | |
5259 | ||
5260 | if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */ | |
5261 | break; | |
5262 | } | |
5263 | ||
9b69cf83 | 5264 | if (i == 0) |
569fa502 | 5265 | break; |
569fa502 DN |
5266 | } |
5267 | } | |
5268 | ||
5269 | return true; | |
5270 | } | |
5271 | ||
5272 | ||
5273 | /* Remove insns from the queue, before they become "ready" with respect | |
6614fd40 | 5274 | to FU latency considerations. */ |
569fa502 | 5275 | |
b8698a0f | 5276 | static int |
569fa502 DN |
5277 | early_queue_to_ready (state_t state, struct ready_list *ready) |
5278 | { | |
ce1ce33a | 5279 | rtx_insn *insn; |
3dc99c19 DM |
5280 | rtx_insn_list *link; |
5281 | rtx_insn_list *next_link; | |
5282 | rtx_insn_list *prev_link; | |
569fa502 DN |
5283 | bool move_to_ready; |
5284 | int cost; | |
5285 | state_t temp_state = alloca (dfa_state_size); | |
5286 | int stalls; | |
5287 | int insns_removed = 0; | |
5288 | ||
5289 | /* | |
b8698a0f L |
5290 | Flag '-fsched-stalled-insns=X' determines the aggressiveness of this |
5291 | function: | |
569fa502 | 5292 | |
b8698a0f | 5293 | X == 0: There is no limit on how many queued insns can be removed |
569fa502 DN |
5294 | prematurely. (flag_sched_stalled_insns = -1). |
5295 | ||
b8698a0f | 5296 | X >= 1: Only X queued insns can be removed prematurely in each |
569fa502 DN |
5297 | invocation. (flag_sched_stalled_insns = X). |
5298 | ||
5299 | Otherwise: Early queue removal is disabled. | |
5300 | (flag_sched_stalled_insns = 0) | |
5301 | */ | |
5302 | ||
b8698a0f | 5303 | if (! flag_sched_stalled_insns) |
569fa502 DN |
5304 | return 0; |
5305 | ||
fa0aee89 | 5306 | for (stalls = 0; stalls <= max_insn_queue_index; stalls++) |
569fa502 DN |
5307 | { |
5308 | if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)])) | |
5309 | { | |
5310 | if (sched_verbose > 6) | |
5311 | fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls); | |
5312 | ||
5313 | prev_link = 0; | |
5314 | while (link) | |
5315 | { | |
3dc99c19 DM |
5316 | next_link = link->next (); |
5317 | insn = link->insn (); | |
569fa502 DN |
5318 | if (insn && sched_verbose > 6) |
5319 | print_rtl_single (sched_dump, insn); | |
5320 | ||
5321 | memcpy (temp_state, state, dfa_state_size); | |
b8698a0f | 5322 | if (recog_memoized (insn) < 0) |
569fa502 DN |
5323 | /* non-negative to indicate that it's not ready |
5324 | to avoid infinite Q->R->Q->R... */ | |
5325 | cost = 0; | |
5326 | else | |
5327 | cost = state_transition (temp_state, insn); | |
5328 | ||
5329 | if (sched_verbose >= 6) | |
5330 | fprintf (sched_dump, "transition cost = %d\n", cost); | |
5331 | ||
5332 | move_to_ready = false; | |
b8698a0f | 5333 | if (cost < 0) |
569fa502 DN |
5334 | { |
5335 | move_to_ready = ok_for_early_queue_removal (insn); | |
5336 | if (move_to_ready == true) | |
5337 | { | |
5338 | /* move from Q to R */ | |
5339 | q_size -= 1; | |
63f54b1a | 5340 | ready_add (ready, insn, false); |
569fa502 | 5341 | |
b8698a0f | 5342 | if (prev_link) |
569fa502 DN |
5343 | XEXP (prev_link, 1) = next_link; |
5344 | else | |
5345 | insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link; | |
5346 | ||
5347 | free_INSN_LIST_node (link); | |
5348 | ||
5349 | if (sched_verbose >= 2) | |
5350 | fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n", | |
5351 | (*current_sched_info->print_insn) (insn, 0)); | |
5352 | ||
5353 | insns_removed++; | |
5354 | if (insns_removed == flag_sched_stalled_insns) | |
63f54b1a MK |
5355 | /* Remove no more than flag_sched_stalled_insns insns |
5356 | from Q at a time. */ | |
569fa502 DN |
5357 | return insns_removed; |
5358 | } | |
5359 | } | |
5360 | ||
5361 | if (move_to_ready == false) | |
5362 | prev_link = link; | |
5363 | ||
5364 | link = next_link; | |
5365 | } /* while link */ | |
b8698a0f | 5366 | } /* if link */ |
569fa502 DN |
5367 | |
5368 | } /* for stalls.. */ | |
5369 | ||
b8698a0f | 5370 | return insns_removed; |
569fa502 DN |
5371 | } |
5372 | ||
5373 | ||
d5c67efd MK |
5374 | /* Print the ready list for debugging purposes. |
5375 | If READY_TRY is non-zero then only print insns that max_issue | |
5376 | will consider. */ | |
b4ead7d4 | 5377 | static void |
4960a0cb | 5378 | debug_ready_list_1 (struct ready_list *ready, signed char *ready_try) |
1708fd40 | 5379 | { |
ce1ce33a | 5380 | rtx_insn **p; |
b4ead7d4 | 5381 | int i; |
1708fd40 | 5382 | |
b4ead7d4 | 5383 | if (ready->n_ready == 0) |
fae15c93 VM |
5384 | { |
5385 | fprintf (sched_dump, "\n"); | |
5386 | return; | |
5387 | } | |
1708fd40 | 5388 | |
b4ead7d4 BS |
5389 | p = ready_lastpos (ready); |
5390 | for (i = 0; i < ready->n_ready; i++) | |
ce18efcb | 5391 | { |
d5c67efd MK |
5392 | if (ready_try != NULL && ready_try[ready->n_ready - i - 1]) |
5393 | continue; | |
5394 | ||
ce18efcb VM |
5395 | fprintf (sched_dump, " %s:%d", |
5396 | (*current_sched_info->print_insn) (p[i], 0), | |
5397 | INSN_LUID (p[i])); | |
7a1aca9c | 5398 | if (sched_pressure != SCHED_PRESSURE_NONE) |
ce18efcb VM |
5399 | fprintf (sched_dump, "(cost=%d", |
5400 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i])); | |
d5c67efd | 5401 | fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i])); |
ce18efcb VM |
5402 | if (INSN_TICK (p[i]) > clock_var) |
5403 | fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var); | |
b75ae7f4 MK |
5404 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
5405 | fprintf (sched_dump, ":idx=%d", | |
5406 | model_index (p[i])); | |
7a1aca9c | 5407 | if (sched_pressure != SCHED_PRESSURE_NONE) |
ce18efcb VM |
5408 | fprintf (sched_dump, ")"); |
5409 | } | |
b4ead7d4 BS |
5410 | fprintf (sched_dump, "\n"); |
5411 | } | |
1708fd40 | 5412 | |
d5c67efd MK |
5413 | /* Print the ready list. Callable from debugger. */ |
5414 | static void | |
5415 | debug_ready_list (struct ready_list *ready) | |
5416 | { | |
5417 | debug_ready_list_1 (ready, NULL); | |
5418 | } | |
5419 | ||
e67271bd RH |
5420 | /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn |
5421 | NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb | |
5422 | replaces the epilogue note in the correct basic block. */ | |
e855c69d | 5423 | void |
b2908ba6 | 5424 | reemit_notes (rtx_insn *insn) |
8c660648 | 5425 | { |
b2908ba6 DM |
5426 | rtx note; |
5427 | rtx_insn *last = insn; | |
8c660648 | 5428 | |
8c660648 JL |
5429 | for (note = REG_NOTES (insn); note; note = XEXP (note, 1)) |
5430 | { | |
c46a37c4 | 5431 | if (REG_NOTE_KIND (note) == REG_SAVE_NOTE) |
8c660648 | 5432 | { |
81f40b79 | 5433 | enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0)); |
b3b42a4d | 5434 | |
63f4a88e JH |
5435 | last = emit_note_before (note_type, last); |
5436 | remove_note (insn, note); | |
8c660648 JL |
5437 | } |
5438 | } | |
8c660648 JL |
5439 | } |
5440 | ||
496d7bb0 MK |
5441 | /* Move INSN. Reemit notes if needed. Update CFG, if needed. */ |
5442 | static void | |
e6eda746 | 5443 | move_insn (rtx_insn *insn, rtx_insn *last, rtx nt) |
496d7bb0 | 5444 | { |
496d7bb0 MK |
5445 | if (PREV_INSN (insn) != last) |
5446 | { | |
5447 | basic_block bb; | |
ce1ce33a | 5448 | rtx_insn *note; |
496d7bb0 | 5449 | int jump_p = 0; |
8c660648 | 5450 | |
496d7bb0 | 5451 | bb = BLOCK_FOR_INSN (insn); |
b8698a0f | 5452 | |
496d7bb0 | 5453 | /* BB_HEAD is either LABEL or NOTE. */ |
b8698a0f | 5454 | gcc_assert (BB_HEAD (bb) != insn); |
496d7bb0 MK |
5455 | |
5456 | if (BB_END (bb) == insn) | |
5457 | /* If this is last instruction in BB, move end marker one | |
5458 | instruction up. */ | |
5459 | { | |
5460 | /* Jumps are always placed at the end of basic block. */ | |
5461 | jump_p = control_flow_insn_p (insn); | |
5462 | ||
5463 | gcc_assert (!jump_p | |
e855c69d | 5464 | || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS) |
d7bfd907 | 5465 | && IS_SPECULATION_BRANCHY_CHECK_P (insn)) |
e855c69d AB |
5466 | || (common_sched_info->sched_pass_id |
5467 | == SCHED_EBB_PASS)); | |
b8698a0f | 5468 | |
496d7bb0 MK |
5469 | gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb); |
5470 | ||
1130d5e3 | 5471 | BB_END (bb) = PREV_INSN (insn); |
496d7bb0 | 5472 | } |
8c660648 | 5473 | |
496d7bb0 | 5474 | gcc_assert (BB_END (bb) != last); |
c9e03727 | 5475 | |
496d7bb0 MK |
5476 | if (jump_p) |
5477 | /* We move the block note along with jump. */ | |
5478 | { | |
e855c69d | 5479 | gcc_assert (nt); |
496d7bb0 MK |
5480 | |
5481 | note = NEXT_INSN (insn); | |
5482 | while (NOTE_NOT_BB_P (note) && note != nt) | |
5483 | note = NEXT_INSN (note); | |
5484 | ||
5485 | if (note != nt | |
5486 | && (LABEL_P (note) | |
5487 | || BARRIER_P (note))) | |
5488 | note = NEXT_INSN (note); | |
b8698a0f | 5489 | |
496d7bb0 MK |
5490 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
5491 | } | |
5492 | else | |
5493 | note = insn; | |
5494 | ||
0f82e5c9 DM |
5495 | SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note); |
5496 | SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn); | |
496d7bb0 | 5497 | |
0f82e5c9 DM |
5498 | SET_NEXT_INSN (note) = NEXT_INSN (last); |
5499 | SET_PREV_INSN (NEXT_INSN (last)) = note; | |
c9e03727 | 5500 | |
0f82e5c9 DM |
5501 | SET_NEXT_INSN (last) = insn; |
5502 | SET_PREV_INSN (insn) = last; | |
496d7bb0 MK |
5503 | |
5504 | bb = BLOCK_FOR_INSN (last); | |
5505 | ||
5506 | if (jump_p) | |
5507 | { | |
5508 | fix_jump_move (insn); | |
5509 | ||
5510 | if (BLOCK_FOR_INSN (insn) != bb) | |
5511 | move_block_after_check (insn); | |
5512 | ||
5513 | gcc_assert (BB_END (bb) == last); | |
5514 | } | |
5515 | ||
63642d5a | 5516 | df_insn_change_bb (insn, bb); |
b8698a0f | 5517 | |
496d7bb0 MK |
5518 | /* Update BB_END, if needed. */ |
5519 | if (BB_END (bb) == last) | |
1130d5e3 | 5520 | BB_END (bb) = insn; |
496d7bb0 | 5521 | } |
58fb7809 | 5522 | |
b8698a0f | 5523 | SCHED_GROUP_P (insn) = 0; |
8c660648 JL |
5524 | } |
5525 | ||
356c23b3 MK |
5526 | /* Return true if scheduling INSN will finish current clock cycle. */ |
5527 | static bool | |
ce1ce33a | 5528 | insn_finishes_cycle_p (rtx_insn *insn) |
356c23b3 MK |
5529 | { |
5530 | if (SCHED_GROUP_P (insn)) | |
5531 | /* After issuing INSN, rest of the sched_group will be forced to issue | |
5532 | in order. Don't make any plans for the rest of cycle. */ | |
5533 | return true; | |
5534 | ||
5535 | /* Finishing the block will, apparently, finish the cycle. */ | |
5536 | if (current_sched_info->insn_finishes_block_p | |
5537 | && current_sched_info->insn_finishes_block_p (insn)) | |
5538 | return true; | |
5539 | ||
5540 | return false; | |
5541 | } | |
5542 | ||
613743a2 KT |
5543 | /* Helper for autopref_multipass_init. Given a SET in PAT and whether |
5544 | we're expecting a memory WRITE or not, check that the insn is relevant to | |
5545 | the autoprefetcher modelling code. Return true iff that is the case. | |
5546 | If it is relevant, record the base register of the memory op in BASE and | |
5547 | the offset in OFFSET. */ | |
5548 | ||
5549 | static bool | |
5550 | analyze_set_insn_for_autopref (rtx pat, bool write, rtx *base, int *offset) | |
5551 | { | |
5552 | if (GET_CODE (pat) != SET) | |
5553 | return false; | |
5554 | ||
5555 | rtx mem = write ? SET_DEST (pat) : SET_SRC (pat); | |
5556 | if (!MEM_P (mem)) | |
5557 | return false; | |
5558 | ||
5559 | struct address_info info; | |
5560 | decompose_mem_address (&info, mem); | |
5561 | ||
5562 | /* TODO: Currently only (base+const) addressing is supported. */ | |
5563 | if (info.base == NULL || !REG_P (*info.base) | |
5564 | || (info.disp != NULL && !CONST_INT_P (*info.disp))) | |
5565 | return false; | |
5566 | ||
5567 | *base = *info.base; | |
5568 | *offset = info.disp ? INTVAL (*info.disp) : 0; | |
5569 | return true; | |
5570 | } | |
5571 | ||
340c7904 MK |
5572 | /* Functions to model cache auto-prefetcher. |
5573 | ||
5574 | Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate | |
5575 | memory prefetches if it sees instructions with consequitive memory accesses | |
5576 | in the instruction stream. Details of such hardware units are not published, | |
5577 | so we can only guess what exactly is going on there. | |
5578 | In the scheduler, we model abstract auto-prefetcher. If there are memory | |
5579 | insns in the ready list (or the queue) that have same memory base, but | |
5580 | different offsets, then we delay the insns with larger offsets until insns | |
5581 | with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH | |
5582 | is "1", then we look at the ready list; if it is N>1, then we also look | |
5583 | through N-1 queue entries. | |
5584 | If the param is N>=0, then rank_for_schedule will consider auto-prefetching | |
5585 | among its heuristics. | |
5586 | Param value of "-1" disables modelling of the auto-prefetcher. */ | |
5587 | ||
5588 | /* Initialize autoprefetcher model data for INSN. */ | |
5589 | static void | |
5590 | autopref_multipass_init (const rtx_insn *insn, int write) | |
5591 | { | |
5592 | autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write]; | |
5593 | ||
5594 | gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED); | |
5595 | data->base = NULL_RTX; | |
f9f43fb3 | 5596 | data->offset = 0; |
340c7904 MK |
5597 | /* Set insn entry initialized, but not relevant for auto-prefetcher. */ |
5598 | data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT; | |
5599 | ||
613743a2 KT |
5600 | rtx pat = PATTERN (insn); |
5601 | ||
5602 | /* We have a multi-set insn like a load-multiple or store-multiple. | |
5603 | We care about these as long as all the memory ops inside the PARALLEL | |
5604 | have the same base register. We care about the minimum and maximum | |
5605 | offsets from that base but don't check for the order of those offsets | |
5606 | within the PARALLEL insn itself. */ | |
5607 | if (GET_CODE (pat) == PARALLEL) | |
5608 | { | |
5609 | int n_elems = XVECLEN (pat, 0); | |
5610 | ||
f9f43fb3 WD |
5611 | int i, offset; |
5612 | rtx base, prev_base = NULL_RTX; | |
5613 | int min_offset = INT_MAX; | |
613743a2 KT |
5614 | |
5615 | for (i = 0; i < n_elems; i++) | |
5616 | { | |
5617 | rtx set = XVECEXP (pat, 0, i); | |
5618 | if (GET_CODE (set) != SET) | |
5619 | return; | |
5620 | ||
613743a2 KT |
5621 | if (!analyze_set_insn_for_autopref (set, write, &base, &offset)) |
5622 | return; | |
5623 | ||
613743a2 KT |
5624 | /* Ensure that all memory operations in the PARALLEL use the same |
5625 | base register. */ | |
f9f43fb3 | 5626 | if (i > 0 && REGNO (base) != REGNO (prev_base)) |
613743a2 | 5627 | return; |
f9f43fb3 WD |
5628 | prev_base = base; |
5629 | min_offset = MIN (min_offset, offset); | |
613743a2 KT |
5630 | } |
5631 | ||
f9f43fb3 WD |
5632 | /* If we reached here then we have a valid PARALLEL of multiple memory ops |
5633 | with prev_base as the base and min_offset containing the offset. */ | |
613743a2 KT |
5634 | gcc_assert (prev_base); |
5635 | data->base = prev_base; | |
f9f43fb3 | 5636 | data->offset = min_offset; |
613743a2 | 5637 | data->status = AUTOPREF_MULTIPASS_DATA_NORMAL; |
613743a2 KT |
5638 | return; |
5639 | } | |
5640 | ||
5641 | /* Otherwise this is a single set memory operation. */ | |
340c7904 MK |
5642 | rtx set = single_set (insn); |
5643 | if (set == NULL_RTX) | |
5644 | return; | |
5645 | ||
613743a2 | 5646 | if (!analyze_set_insn_for_autopref (set, write, &data->base, |
f9f43fb3 | 5647 | &data->offset)) |
340c7904 MK |
5648 | return; |
5649 | ||
613743a2 KT |
5650 | /* This insn is relevant for the auto-prefetcher. |
5651 | The base and offset fields will have been filled in the | |
5652 | analyze_set_insn_for_autopref call above. */ | |
5653 | data->status = AUTOPREF_MULTIPASS_DATA_NORMAL; | |
5654 | } | |
340c7904 | 5655 | |
340c7904 MK |
5656 | /* Helper function for rank_for_schedule sorting. */ |
5657 | static int | |
5658 | autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2) | |
5659 | { | |
ab90c27b AM |
5660 | int r = 0; |
5661 | for (int write = 0; write < 2 && !r; ++write) | |
340c7904 MK |
5662 | { |
5663 | autopref_multipass_data_t data1 | |
5664 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write]; | |
5665 | autopref_multipass_data_t data2 | |
5666 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write]; | |
5667 | ||
5668 | if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5669 | autopref_multipass_init (insn1, write); | |
340c7904 MK |
5670 | |
5671 | if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5672 | autopref_multipass_init (insn2, write); | |
340c7904 | 5673 | |
ab90c27b AM |
5674 | int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT; |
5675 | int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT; | |
340c7904 | 5676 | |
ab90c27b | 5677 | if (!irrel1 && !irrel2) |
f9f43fb3 | 5678 | r = data1->offset - data2->offset; |
ab90c27b AM |
5679 | else |
5680 | r = irrel2 - irrel1; | |
340c7904 MK |
5681 | } |
5682 | ||
ab90c27b | 5683 | return r; |
340c7904 MK |
5684 | } |
5685 | ||
5686 | /* True if header of debug dump was printed. */ | |
5687 | static bool autopref_multipass_dfa_lookahead_guard_started_dump_p; | |
5688 | ||
5689 | /* Helper for autopref_multipass_dfa_lookahead_guard. | |
5690 | Return "1" if INSN1 should be delayed in favor of INSN2. */ | |
5691 | static int | |
5692 | autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1, | |
5693 | const rtx_insn *insn2, int write) | |
5694 | { | |
5695 | autopref_multipass_data_t data1 | |
5696 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write]; | |
5697 | autopref_multipass_data_t data2 | |
5698 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write]; | |
5699 | ||
5700 | if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5701 | autopref_multipass_init (insn2, write); | |
5702 | if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT) | |
5703 | return 0; | |
5704 | ||
5705 | if (rtx_equal_p (data1->base, data2->base) | |
f9f43fb3 | 5706 | && data1->offset > data2->offset) |
340c7904 MK |
5707 | { |
5708 | if (sched_verbose >= 2) | |
5709 | { | |
5710 | if (!autopref_multipass_dfa_lookahead_guard_started_dump_p) | |
5711 | { | |
5712 | fprintf (sched_dump, | |
5713 | ";;\t\tnot trying in max_issue due to autoprefetch " | |
5714 | "model: "); | |
5715 | autopref_multipass_dfa_lookahead_guard_started_dump_p = true; | |
5716 | } | |
5717 | ||
5718 | fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2)); | |
5719 | } | |
5720 | ||
5721 | return 1; | |
5722 | } | |
5723 | ||
5724 | return 0; | |
5725 | } | |
5726 | ||
5727 | /* General note: | |
5728 | ||
5729 | We could have also hooked autoprefetcher model into | |
5730 | first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks | |
5731 | to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle | |
5732 | (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets | |
5733 | unblocked). We don't bother about this yet because target of interest | |
5734 | (ARM Cortex-A15) can issue only 1 memory operation per cycle. */ | |
5735 | ||
5736 | /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook. | |
5737 | Return "1" if INSN1 should not be considered in max_issue due to | |
5738 | auto-prefetcher considerations. */ | |
5739 | int | |
5740 | autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index) | |
5741 | { | |
5742 | int r = 0; | |
5743 | ||
df554b0e KT |
5744 | /* Exit early if the param forbids this or if we're not entering here through |
5745 | normal haifa scheduling. This can happen if selective scheduling is | |
5746 | explicitly enabled. */ | |
5747 | if (!insn_queue || PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) <= 0) | |
340c7904 MK |
5748 | return 0; |
5749 | ||
5750 | if (sched_verbose >= 2 && ready_index == 0) | |
5751 | autopref_multipass_dfa_lookahead_guard_started_dump_p = false; | |
5752 | ||
5753 | for (int write = 0; write < 2; ++write) | |
5754 | { | |
5755 | autopref_multipass_data_t data1 | |
5756 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write]; | |
5757 | ||
5758 | if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5759 | autopref_multipass_init (insn1, write); | |
5760 | if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT) | |
5761 | continue; | |
5762 | ||
5763 | if (ready_index == 0 | |
5764 | && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY) | |
5765 | /* We allow only a single delay on priviledged instructions. | |
5766 | Doing otherwise would cause infinite loop. */ | |
5767 | { | |
5768 | if (sched_verbose >= 2) | |
5769 | { | |
5770 | if (!autopref_multipass_dfa_lookahead_guard_started_dump_p) | |
5771 | { | |
5772 | fprintf (sched_dump, | |
5773 | ";;\t\tnot trying in max_issue due to autoprefetch " | |
5774 | "model: "); | |
5775 | autopref_multipass_dfa_lookahead_guard_started_dump_p = true; | |
5776 | } | |
5777 | ||
5778 | fprintf (sched_dump, " *%d*", INSN_UID (insn1)); | |
5779 | } | |
5780 | continue; | |
5781 | } | |
5782 | ||
5783 | for (int i2 = 0; i2 < ready.n_ready; ++i2) | |
5784 | { | |
5785 | rtx_insn *insn2 = get_ready_element (i2); | |
5786 | if (insn1 == insn2) | |
5787 | continue; | |
5788 | r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write); | |
5789 | if (r) | |
5790 | { | |
5791 | if (ready_index == 0) | |
5792 | { | |
5793 | r = -1; | |
5794 | data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY; | |
5795 | } | |
5796 | goto finish; | |
5797 | } | |
5798 | } | |
5799 | ||
5800 | if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) == 1) | |
5801 | continue; | |
5802 | ||
5803 | /* Everything from the current queue slot should have been moved to | |
5804 | the ready list. */ | |
5805 | gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX); | |
5806 | ||
5807 | int n_stalls = PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) - 1; | |
5808 | if (n_stalls > max_insn_queue_index) | |
5809 | n_stalls = max_insn_queue_index; | |
5810 | ||
5811 | for (int stalls = 1; stalls <= n_stalls; ++stalls) | |
5812 | { | |
5813 | for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]; | |
5814 | link != NULL_RTX; | |
5815 | link = link->next ()) | |
5816 | { | |
5817 | rtx_insn *insn2 = link->insn (); | |
5818 | r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, | |
5819 | write); | |
5820 | if (r) | |
5821 | { | |
5822 | /* Queue INSN1 until INSN2 can issue. */ | |
5823 | r = -stalls; | |
5824 | if (ready_index == 0) | |
5825 | data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY; | |
5826 | goto finish; | |
5827 | } | |
5828 | } | |
5829 | } | |
5830 | } | |
5831 | ||
5832 | finish: | |
5833 | if (sched_verbose >= 2 | |
5834 | && autopref_multipass_dfa_lookahead_guard_started_dump_p | |
5835 | && (ready_index == ready.n_ready - 1 || r < 0)) | |
5836 | /* This does not /always/ trigger. We don't output EOL if the last | |
5837 | insn is not recognized (INSN_CODE < 0) and lookahead_guard is not | |
5838 | called. We can live with this. */ | |
5839 | fprintf (sched_dump, "\n"); | |
5840 | ||
5841 | return r; | |
5842 | } | |
5843 | ||
894fd6f2 MK |
5844 | /* Define type for target data used in multipass scheduling. */ |
5845 | #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T | |
5846 | # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int | |
5847 | #endif | |
5848 | typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t; | |
5849 | ||
30028c85 VM |
5850 | /* The following structure describe an entry of the stack of choices. */ |
5851 | struct choice_entry | |
5852 | { | |
5853 | /* Ordinal number of the issued insn in the ready queue. */ | |
5854 | int index; | |
5855 | /* The number of the rest insns whose issues we should try. */ | |
5856 | int rest; | |
5857 | /* The number of issued essential insns. */ | |
5858 | int n; | |
5859 | /* State after issuing the insn. */ | |
5860 | state_t state; | |
894fd6f2 MK |
5861 | /* Target-specific data. */ |
5862 | first_cycle_multipass_data_t target_data; | |
30028c85 VM |
5863 | }; |
5864 | ||
5865 | /* The following array is used to implement a stack of choices used in | |
5866 | function max_issue. */ | |
5867 | static struct choice_entry *choice_stack; | |
5868 | ||
e855c69d AB |
5869 | /* This holds the value of the target dfa_lookahead hook. */ |
5870 | int dfa_lookahead; | |
30028c85 | 5871 | |
880efc46 VM |
5872 | /* The following variable value is maximal number of tries of issuing |
5873 | insns for the first cycle multipass insn scheduling. We define | |
5874 | this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not | |
5875 | need this constraint if all real insns (with non-negative codes) | |
5876 | had reservations because in this case the algorithm complexity is | |
5877 | O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions | |
5878 | might be incomplete and such insn might occur. For such | |
5879 | descriptions, the complexity of algorithm (without the constraint) | |
5880 | could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */ | |
5881 | static int max_lookahead_tries; | |
5882 | ||
fae15c93 VM |
5883 | /* The following function returns maximal (or close to maximal) number |
5884 | of insns which can be issued on the same cycle and one of which | |
30028c85 | 5885 | insns is insns with the best rank (the first insn in READY). To |
fae15c93 VM |
5886 | make this function tries different samples of ready insns. READY |
5887 | is current queue `ready'. Global array READY_TRY reflects what | |
84fabdde | 5888 | insns are already issued in this try. The function stops immediately, |
496d7bb0 MK |
5889 | if it reached the such a solution, that all instruction can be issued. |
5890 | INDEX will contain index of the best insn in READY. The following | |
e855c69d AB |
5891 | function is used only for first cycle multipass scheduling. |
5892 | ||
5893 | PRIVILEGED_N >= 0 | |
5894 | ||
5895 | This function expects recognized insns only. All USEs, | |
5896 | CLOBBERs, etc must be filtered elsewhere. */ | |
5897 | int | |
5898 | max_issue (struct ready_list *ready, int privileged_n, state_t state, | |
894fd6f2 | 5899 | bool first_cycle_insn_p, int *index) |
fae15c93 | 5900 | { |
84fabdde | 5901 | int n, i, all, n_ready, best, delay, tries_num; |
e855c69d | 5902 | int more_issue; |
30028c85 | 5903 | struct choice_entry *top; |
ce1ce33a | 5904 | rtx_insn *insn; |
fae15c93 | 5905 | |
b16abbcb BC |
5906 | if (sched_fusion) |
5907 | return 0; | |
5908 | ||
e855c69d AB |
5909 | n_ready = ready->n_ready; |
5910 | gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0 | |
5911 | && privileged_n <= n_ready); | |
5912 | ||
5913 | /* Init MAX_LOOKAHEAD_TRIES. */ | |
8aec463b | 5914 | if (max_lookahead_tries == 0) |
e855c69d | 5915 | { |
e855c69d AB |
5916 | max_lookahead_tries = 100; |
5917 | for (i = 0; i < issue_rate; i++) | |
5918 | max_lookahead_tries *= dfa_lookahead; | |
5919 | } | |
5920 | ||
5921 | /* Init max_points. */ | |
e855c69d | 5922 | more_issue = issue_rate - cycle_issued_insns; |
646e6f41 | 5923 | gcc_assert (more_issue >= 0); |
e855c69d | 5924 | |
e855c69d | 5925 | /* The number of the issued insns in the best solution. */ |
fae15c93 | 5926 | best = 0; |
e855c69d | 5927 | |
30028c85 | 5928 | top = choice_stack; |
e855c69d AB |
5929 | |
5930 | /* Set initial state of the search. */ | |
5931 | memcpy (top->state, state, dfa_state_size); | |
5932 | top->rest = dfa_lookahead; | |
30028c85 | 5933 | top->n = 0; |
894fd6f2 MK |
5934 | if (targetm.sched.first_cycle_multipass_begin) |
5935 | targetm.sched.first_cycle_multipass_begin (&top->target_data, | |
5936 | ready_try, n_ready, | |
5937 | first_cycle_insn_p); | |
e855c69d AB |
5938 | |
5939 | /* Count the number of the insns to search among. */ | |
30028c85 | 5940 | for (all = i = 0; i < n_ready; i++) |
fae15c93 | 5941 | if (!ready_try [i]) |
30028c85 | 5942 | all++; |
e855c69d | 5943 | |
d5c67efd MK |
5944 | if (sched_verbose >= 2) |
5945 | { | |
5946 | fprintf (sched_dump, ";;\t\tmax_issue among %d insns:", all); | |
5947 | debug_ready_list_1 (ready, ready_try); | |
5948 | } | |
5949 | ||
e855c69d | 5950 | /* I is the index of the insn to try next. */ |
30028c85 | 5951 | i = 0; |
880efc46 | 5952 | tries_num = 0; |
30028c85 VM |
5953 | for (;;) |
5954 | { | |
e855c69d AB |
5955 | if (/* If we've reached a dead end or searched enough of what we have |
5956 | been asked... */ | |
5957 | top->rest == 0 | |
84fabdde JZ |
5958 | /* or have nothing else to try... */ |
5959 | || i >= n_ready | |
5960 | /* or should not issue more. */ | |
5961 | || top->n >= more_issue) | |
30028c85 | 5962 | { |
e855c69d AB |
5963 | /* ??? (... || i == n_ready). */ |
5964 | gcc_assert (i <= n_ready); | |
5965 | ||
84fabdde JZ |
5966 | /* We should not issue more than issue_rate instructions. */ |
5967 | gcc_assert (top->n <= more_issue); | |
5968 | ||
30028c85 VM |
5969 | if (top == choice_stack) |
5970 | break; | |
e855c69d AB |
5971 | |
5972 | if (best < top - choice_stack) | |
30028c85 | 5973 | { |
e855c69d AB |
5974 | if (privileged_n) |
5975 | { | |
5976 | n = privileged_n; | |
5977 | /* Try to find issued privileged insn. */ | |
e84a58ff EB |
5978 | while (n && !ready_try[--n]) |
5979 | ; | |
e855c69d AB |
5980 | } |
5981 | ||
5982 | if (/* If all insns are equally good... */ | |
5983 | privileged_n == 0 | |
5984 | /* Or a privileged insn will be issued. */ | |
5985 | || ready_try[n]) | |
5986 | /* Then we have a solution. */ | |
5987 | { | |
5988 | best = top - choice_stack; | |
5989 | /* This is the index of the insn issued first in this | |
5990 | solution. */ | |
5991 | *index = choice_stack [1].index; | |
84fabdde | 5992 | if (top->n == more_issue || best == all) |
e855c69d AB |
5993 | break; |
5994 | } | |
30028c85 | 5995 | } |
e855c69d AB |
5996 | |
5997 | /* Set ready-list index to point to the last insn | |
5998 | ('i++' below will advance it to the next insn). */ | |
30028c85 | 5999 | i = top->index; |
e855c69d AB |
6000 | |
6001 | /* Backtrack. */ | |
30028c85 | 6002 | ready_try [i] = 0; |
894fd6f2 MK |
6003 | |
6004 | if (targetm.sched.first_cycle_multipass_backtrack) | |
6005 | targetm.sched.first_cycle_multipass_backtrack (&top->target_data, | |
6006 | ready_try, n_ready); | |
6007 | ||
30028c85 | 6008 | top--; |
e855c69d | 6009 | memcpy (state, top->state, dfa_state_size); |
30028c85 VM |
6010 | } |
6011 | else if (!ready_try [i]) | |
6012 | { | |
880efc46 VM |
6013 | tries_num++; |
6014 | if (tries_num > max_lookahead_tries) | |
6015 | break; | |
30028c85 | 6016 | insn = ready_element (ready, i); |
e855c69d | 6017 | delay = state_transition (state, insn); |
30028c85 VM |
6018 | if (delay < 0) |
6019 | { | |
356c23b3 MK |
6020 | if (state_dead_lock_p (state) |
6021 | || insn_finishes_cycle_p (insn)) | |
9d701248 BS |
6022 | /* We won't issue any more instructions in the next |
6023 | choice_state. */ | |
30028c85 VM |
6024 | top->rest = 0; |
6025 | else | |
6026 | top->rest--; | |
e855c69d | 6027 | |
30028c85 | 6028 | n = top->n; |
e855c69d | 6029 | if (memcmp (top->state, state, dfa_state_size) != 0) |
84fabdde | 6030 | n++; |
e855c69d AB |
6031 | |
6032 | /* Advance to the next choice_entry. */ | |
30028c85 | 6033 | top++; |
e855c69d AB |
6034 | /* Initialize it. */ |
6035 | top->rest = dfa_lookahead; | |
30028c85 VM |
6036 | top->index = i; |
6037 | top->n = n; | |
e855c69d | 6038 | memcpy (top->state, state, dfa_state_size); |
30028c85 | 6039 | ready_try [i] = 1; |
894fd6f2 MK |
6040 | |
6041 | if (targetm.sched.first_cycle_multipass_issue) | |
6042 | targetm.sched.first_cycle_multipass_issue (&top->target_data, | |
6043 | ready_try, n_ready, | |
6044 | insn, | |
6045 | &((top - 1) | |
6046 | ->target_data)); | |
6047 | ||
30028c85 VM |
6048 | i = -1; |
6049 | } | |
6050 | } | |
e855c69d AB |
6051 | |
6052 | /* Increase ready-list index. */ | |
30028c85 VM |
6053 | i++; |
6054 | } | |
496d7bb0 | 6055 | |
894fd6f2 MK |
6056 | if (targetm.sched.first_cycle_multipass_end) |
6057 | targetm.sched.first_cycle_multipass_end (best != 0 | |
6058 | ? &choice_stack[1].target_data | |
6059 | : NULL); | |
6060 | ||
e855c69d | 6061 | /* Restore the original state of the DFA. */ |
b8698a0f | 6062 | memcpy (state, choice_stack->state, dfa_state_size); |
e855c69d | 6063 | |
fae15c93 VM |
6064 | return best; |
6065 | } | |
6066 | ||
6067 | /* The following function chooses insn from READY and modifies | |
e855c69d | 6068 | READY. The following function is used only for first |
b631c5f7 SP |
6069 | cycle multipass scheduling. |
6070 | Return: | |
6071 | -1 if cycle should be advanced, | |
6072 | 0 if INSN_PTR is set to point to the desirable insn, | |
6073 | 1 if choose_ready () should be restarted without advancing the cycle. */ | |
6074 | static int | |
894fd6f2 | 6075 | choose_ready (struct ready_list *ready, bool first_cycle_insn_p, |
ce1ce33a | 6076 | rtx_insn **insn_ptr) |
fae15c93 | 6077 | { |
b631c5f7 SP |
6078 | if (dbg_cnt (sched_insn) == false) |
6079 | { | |
06bc63d1 MK |
6080 | if (nonscheduled_insns_begin == NULL_RTX) |
6081 | nonscheduled_insns_begin = current_sched_info->prev_head; | |
6082 | ||
ce1ce33a | 6083 | rtx_insn *insn = first_nonscheduled_insn (); |
b631c5f7 SP |
6084 | |
6085 | if (QUEUE_INDEX (insn) == QUEUE_READY) | |
6086 | /* INSN is in the ready_list. */ | |
6087 | { | |
6088 | ready_remove_insn (insn); | |
6089 | *insn_ptr = insn; | |
6090 | return 0; | |
6091 | } | |
6092 | ||
6093 | /* INSN is in the queue. Advance cycle to move it to the ready list. */ | |
06bc63d1 | 6094 | gcc_assert (QUEUE_INDEX (insn) >= 0); |
b631c5f7 SP |
6095 | return -1; |
6096 | } | |
6097 | ||
8aec463b | 6098 | if (dfa_lookahead <= 0 || SCHED_GROUP_P (ready_element (ready, 0)) |
b5b8b0ac | 6099 | || DEBUG_INSN_P (ready_element (ready, 0))) |
b631c5f7 | 6100 | { |
ac44248e | 6101 | if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON)) |
7942e47e RY |
6102 | *insn_ptr = ready_remove_first_dispatch (ready); |
6103 | else | |
6104 | *insn_ptr = ready_remove_first (ready); | |
6105 | ||
b631c5f7 SP |
6106 | return 0; |
6107 | } | |
fae15c93 VM |
6108 | else |
6109 | { | |
4960a0cb | 6110 | /* Try to choose the best insn. */ |
16d83dd6 | 6111 | int index = 0, i; |
ce1ce33a | 6112 | rtx_insn *insn; |
b8698a0f | 6113 | |
30028c85 VM |
6114 | insn = ready_element (ready, 0); |
6115 | if (INSN_CODE (insn) < 0) | |
b631c5f7 SP |
6116 | { |
6117 | *insn_ptr = ready_remove_first (ready); | |
6118 | return 0; | |
6119 | } | |
496d7bb0 | 6120 | |
4960a0cb MK |
6121 | /* Filter the search space. */ |
6122 | for (i = 0; i < ready->n_ready; i++) | |
496d7bb0 | 6123 | { |
4960a0cb | 6124 | ready_try[i] = 0; |
496d7bb0 | 6125 | |
4960a0cb | 6126 | insn = ready_element (ready, i); |
496d7bb0 | 6127 | |
4960a0cb MK |
6128 | /* If this insn is recognizable we should have already |
6129 | recognized it earlier. | |
6130 | ??? Not very clear where this is supposed to be done. | |
6131 | See dep_cost_1. */ | |
6132 | gcc_checking_assert (INSN_CODE (insn) >= 0 | |
6133 | || recog_memoized (insn) < 0); | |
6134 | if (INSN_CODE (insn) < 0) | |
6135 | { | |
6136 | /* Non-recognized insns at position 0 are handled above. */ | |
6137 | gcc_assert (i > 0); | |
6138 | ready_try[i] = 1; | |
6139 | continue; | |
6140 | } | |
6141 | ||
6142 | if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard) | |
6143 | { | |
6144 | ready_try[i] | |
6145 | = (targetm.sched.first_cycle_multipass_dfa_lookahead_guard | |
6146 | (insn, i)); | |
6147 | ||
6148 | if (ready_try[i] < 0) | |
6149 | /* Queue instruction for several cycles. | |
6150 | We need to restart choose_ready as we have changed | |
6151 | the ready list. */ | |
6152 | { | |
6153 | change_queue_index (insn, -ready_try[i]); | |
6154 | return 1; | |
6155 | } | |
6156 | ||
6157 | /* Make sure that we didn't end up with 0'th insn filtered out. | |
6158 | Don't be tempted to make life easier for backends and just | |
6159 | requeue 0'th insn if (ready_try[0] == 0) and restart | |
6160 | choose_ready. Backends should be very considerate about | |
6161 | requeueing instructions -- especially the highest priority | |
6162 | one at position 0. */ | |
6163 | gcc_assert (ready_try[i] == 0 || i > 0); | |
6164 | if (ready_try[i]) | |
6165 | continue; | |
6166 | } | |
6167 | ||
6168 | gcc_assert (ready_try[i] == 0); | |
6169 | /* INSN made it through the scrutiny of filters! */ | |
6170 | } | |
e855c69d | 6171 | |
894fd6f2 | 6172 | if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0) |
b631c5f7 SP |
6173 | { |
6174 | *insn_ptr = ready_remove_first (ready); | |
9c575182 | 6175 | if (sched_verbose >= 4) |
b8698a0f | 6176 | fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n", |
9c575182 | 6177 | (*current_sched_info->print_insn) (*insn_ptr, 0)); |
b631c5f7 SP |
6178 | return 0; |
6179 | } | |
fae15c93 | 6180 | else |
b631c5f7 | 6181 | { |
b8698a0f | 6182 | if (sched_verbose >= 4) |
e855c69d AB |
6183 | fprintf (sched_dump, ";;\t\tChosen insn : %s\n", |
6184 | (*current_sched_info->print_insn) | |
6185 | (ready_element (ready, index), 0)); | |
b8698a0f | 6186 | |
b631c5f7 SP |
6187 | *insn_ptr = ready_remove (ready, index); |
6188 | return 0; | |
6189 | } | |
fae15c93 VM |
6190 | } |
6191 | } | |
6192 | ||
86014d07 BS |
6193 | /* This function is called when we have successfully scheduled a |
6194 | block. It uses the schedule stored in the scheduled_insns vector | |
6195 | to rearrange the RTL. PREV_HEAD is used as the anchor to which we | |
6196 | append the scheduled insns; TAIL is the insn after the scheduled | |
6197 | block. TARGET_BB is the argument passed to schedule_block. */ | |
6198 | ||
6199 | static void | |
f57aa6b0 | 6200 | commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb) |
86014d07 | 6201 | { |
9b69cf83 | 6202 | unsigned int i; |
ce1ce33a | 6203 | rtx_insn *insn; |
86014d07 BS |
6204 | |
6205 | last_scheduled_insn = prev_head; | |
9b69cf83 | 6206 | for (i = 0; |
9771b263 | 6207 | scheduled_insns.iterate (i, &insn); |
9b69cf83 | 6208 | i++) |
86014d07 | 6209 | { |
86014d07 BS |
6210 | if (control_flow_insn_p (last_scheduled_insn) |
6211 | || current_sched_info->advance_target_bb (*target_bb, insn)) | |
6212 | { | |
6213 | *target_bb = current_sched_info->advance_target_bb (*target_bb, 0); | |
6214 | ||
6215 | if (sched_verbose) | |
6216 | { | |
f57aa6b0 | 6217 | rtx_insn *x; |
86014d07 BS |
6218 | |
6219 | x = next_real_insn (last_scheduled_insn); | |
6220 | gcc_assert (x); | |
6221 | dump_new_block_header (1, *target_bb, x, tail); | |
6222 | } | |
6223 | ||
6224 | last_scheduled_insn = bb_note (*target_bb); | |
6225 | } | |
6226 | ||
6227 | if (current_sched_info->begin_move_insn) | |
6228 | (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn); | |
6229 | move_insn (insn, last_scheduled_insn, | |
6230 | current_sched_info->next_tail); | |
6231 | if (!DEBUG_INSN_P (insn)) | |
6232 | reemit_notes (insn); | |
6233 | last_scheduled_insn = insn; | |
6234 | } | |
6235 | ||
9771b263 | 6236 | scheduled_insns.truncate (0); |
86014d07 BS |
6237 | } |
6238 | ||
9d701248 BS |
6239 | /* Examine all insns on the ready list and queue those which can't be |
6240 | issued in this cycle. TEMP_STATE is temporary scheduler state we | |
6241 | can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns | |
6242 | have been issued for the current cycle, which means it is valid to | |
26965010 BS |
6243 | issue an asm statement. |
6244 | ||
6245 | If SHADOWS_ONLY_P is true, we eliminate all real insns and only | |
06d7e8e7 BS |
6246 | leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true, |
6247 | we only leave insns which have an INSN_EXACT_TICK. */ | |
9d701248 BS |
6248 | |
6249 | static void | |
26965010 | 6250 | prune_ready_list (state_t temp_state, bool first_cycle_insn_p, |
06d7e8e7 | 6251 | bool shadows_only_p, bool modulo_epilogue_p) |
9d701248 | 6252 | { |
05641603 | 6253 | int i, pass; |
b75f962c | 6254 | bool sched_group_found = false; |
e9593da8 | 6255 | int min_cost_group = 0; |
9d701248 | 6256 | |
b16abbcb BC |
6257 | if (sched_fusion) |
6258 | return; | |
6259 | ||
9d701248 BS |
6260 | for (i = 0; i < ready.n_ready; i++) |
6261 | { | |
ce1ce33a | 6262 | rtx_insn *insn = ready_element (&ready, i); |
05641603 | 6263 | if (SCHED_GROUP_P (insn)) |
b75f962c BS |
6264 | { |
6265 | sched_group_found = true; | |
05641603 | 6266 | break; |
b75f962c | 6267 | } |
05641603 | 6268 | } |
b75f962c | 6269 | |
05641603 | 6270 | /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle |
e9593da8 JW |
6271 | such an insn first and note its cost. If at least one SCHED_GROUP_P insn |
6272 | gets queued, then all other insns get queued for one cycle later. */ | |
05641603 BS |
6273 | for (pass = sched_group_found ? 0 : 1; pass < 2; ) |
6274 | { | |
6275 | int n = ready.n_ready; | |
6276 | for (i = 0; i < n; i++) | |
9d701248 | 6277 | { |
ce1ce33a | 6278 | rtx_insn *insn = ready_element (&ready, i); |
05641603 BS |
6279 | int cost = 0; |
6280 | const char *reason = "resource conflict"; | |
26965010 | 6281 | |
05641603 BS |
6282 | if (DEBUG_INSN_P (insn)) |
6283 | continue; | |
6284 | ||
e9593da8 JW |
6285 | if (sched_group_found && !SCHED_GROUP_P (insn) |
6286 | && ((pass == 0) || (min_cost_group >= 1))) | |
05641603 BS |
6287 | { |
6288 | if (pass == 0) | |
6289 | continue; | |
6290 | cost = min_cost_group; | |
6291 | reason = "not in sched group"; | |
6292 | } | |
6293 | else if (modulo_epilogue_p | |
6294 | && INSN_EXACT_TICK (insn) == INVALID_TICK) | |
6295 | { | |
6296 | cost = max_insn_queue_index; | |
6297 | reason = "not an epilogue insn"; | |
6298 | } | |
6299 | else if (shadows_only_p && !SHADOW_P (insn)) | |
26965010 | 6300 | { |
05641603 BS |
6301 | cost = 1; |
6302 | reason = "not a shadow"; | |
6303 | } | |
6304 | else if (recog_memoized (insn) < 0) | |
6305 | { | |
6306 | if (!first_cycle_insn_p | |
6307 | && (GET_CODE (PATTERN (insn)) == ASM_INPUT | |
6308 | || asm_noperands (PATTERN (insn)) >= 0)) | |
6309 | cost = 1; | |
6310 | reason = "asm"; | |
6311 | } | |
7a1aca9c RS |
6312 | else if (sched_pressure != SCHED_PRESSURE_NONE) |
6313 | { | |
6314 | if (sched_pressure == SCHED_PRESSURE_MODEL | |
6315 | && INSN_TICK (insn) <= clock_var) | |
6316 | { | |
6317 | memcpy (temp_state, curr_state, dfa_state_size); | |
6318 | if (state_transition (temp_state, insn) >= 0) | |
6319 | INSN_TICK (insn) = clock_var + 1; | |
6320 | } | |
6321 | cost = 0; | |
6322 | } | |
05641603 BS |
6323 | else |
6324 | { | |
6325 | int delay_cost = 0; | |
6326 | ||
c203e8a7 | 6327 | if (delay_htab) |
26965010 | 6328 | { |
05641603 BS |
6329 | struct delay_pair *delay_entry; |
6330 | delay_entry | |
c203e8a7 TS |
6331 | = delay_htab->find_with_hash (insn, |
6332 | htab_hash_pointer (insn)); | |
05641603 BS |
6333 | while (delay_entry && delay_cost == 0) |
6334 | { | |
6335 | delay_cost = estimate_shadow_tick (delay_entry); | |
6336 | if (delay_cost > max_insn_queue_index) | |
6337 | delay_cost = max_insn_queue_index; | |
6338 | delay_entry = delay_entry->next_same_i1; | |
6339 | } | |
26965010 | 6340 | } |
26965010 | 6341 | |
05641603 BS |
6342 | memcpy (temp_state, curr_state, dfa_state_size); |
6343 | cost = state_transition (temp_state, insn); | |
6344 | if (cost < 0) | |
6345 | cost = 0; | |
6346 | else if (cost == 0) | |
6347 | cost = 1; | |
6348 | if (cost < delay_cost) | |
6349 | { | |
6350 | cost = delay_cost; | |
6351 | reason = "shadow tick"; | |
6352 | } | |
6353 | } | |
6354 | if (cost >= 1) | |
26965010 | 6355 | { |
05641603 BS |
6356 | if (SCHED_GROUP_P (insn) && cost > min_cost_group) |
6357 | min_cost_group = cost; | |
6358 | ready_remove (&ready, i); | |
3f9b5ffb JL |
6359 | /* Normally we'd want to queue INSN for COST cycles. However, |
6360 | if SCHED_GROUP_P is set, then we must ensure that nothing | |
6361 | else comes between INSN and its predecessor. If there is | |
6362 | some other insn ready to fire on the next cycle, then that | |
6363 | invariant would be broken. | |
6364 | ||
6365 | So when SCHED_GROUP_P is set, just queue this insn for a | |
6366 | single cycle. */ | |
6367 | queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason); | |
05641603 BS |
6368 | if (i + 1 < n) |
6369 | break; | |
26965010 | 6370 | } |
9d701248 | 6371 | } |
05641603 BS |
6372 | if (i == n) |
6373 | pass++; | |
9d701248 BS |
6374 | } |
6375 | } | |
6376 | ||
26965010 BS |
6377 | /* Called when we detect that the schedule is impossible. We examine the |
6378 | backtrack queue to find the earliest insn that caused this condition. */ | |
6379 | ||
6380 | static struct haifa_saved_data * | |
6381 | verify_shadows (void) | |
6382 | { | |
6383 | struct haifa_saved_data *save, *earliest_fail = NULL; | |
6384 | for (save = backtrack_queue; save; save = save->next) | |
6385 | { | |
6386 | int t; | |
6387 | struct delay_pair *pair = save->delay_pair; | |
f57aa6b0 | 6388 | rtx_insn *i1 = pair->i1; |
26965010 BS |
6389 | |
6390 | for (; pair; pair = pair->next_same_i1) | |
6391 | { | |
f57aa6b0 | 6392 | rtx_insn *i2 = pair->i2; |
26965010 BS |
6393 | |
6394 | if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED) | |
6395 | continue; | |
6396 | ||
6397 | t = INSN_TICK (i1) + pair_delay (pair); | |
6398 | if (t < clock_var) | |
6399 | { | |
6400 | if (sched_verbose >= 2) | |
6401 | fprintf (sched_dump, | |
6402 | ";;\t\tfailed delay requirements for %d/%d (%d->%d)" | |
6403 | ", not ready\n", | |
6404 | INSN_UID (pair->i1), INSN_UID (pair->i2), | |
6405 | INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2)); | |
6406 | earliest_fail = save; | |
6407 | break; | |
6408 | } | |
6409 | if (QUEUE_INDEX (i2) >= 0) | |
6410 | { | |
6411 | int queued_for = INSN_TICK (i2); | |
6412 | ||
6413 | if (t < queued_for) | |
6414 | { | |
6415 | if (sched_verbose >= 2) | |
6416 | fprintf (sched_dump, | |
6417 | ";;\t\tfailed delay requirements for %d/%d" | |
6418 | " (%d->%d), queued too late\n", | |
6419 | INSN_UID (pair->i1), INSN_UID (pair->i2), | |
6420 | INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2)); | |
6421 | earliest_fail = save; | |
6422 | break; | |
6423 | } | |
6424 | } | |
6425 | } | |
6426 | } | |
6427 | ||
6428 | return earliest_fail; | |
6429 | } | |
6430 | ||
d5c67efd MK |
6431 | /* Print instructions together with useful scheduling information between |
6432 | HEAD and TAIL (inclusive). */ | |
6433 | static void | |
f57aa6b0 | 6434 | dump_insn_stream (rtx_insn *head, rtx_insn *tail) |
d5c67efd MK |
6435 | { |
6436 | fprintf (sched_dump, ";;\t| insn | prio |\n"); | |
6437 | ||
f57aa6b0 DM |
6438 | rtx_insn *next_tail = NEXT_INSN (tail); |
6439 | for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn)) | |
d5c67efd MK |
6440 | { |
6441 | int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn); | |
6442 | const char *pattern = (NOTE_P (insn) | |
6443 | ? "note" | |
6444 | : str_pattern_slim (PATTERN (insn))); | |
6445 | ||
6446 | fprintf (sched_dump, ";;\t| %4d | %4d | %-30s ", | |
6447 | INSN_UID (insn), priority, pattern); | |
6448 | ||
6449 | if (sched_verbose >= 4) | |
6450 | { | |
81e63b63 | 6451 | if (NOTE_P (insn) || LABEL_P (insn) || recog_memoized (insn) < 0) |
d5c67efd MK |
6452 | fprintf (sched_dump, "nothing"); |
6453 | else | |
6454 | print_reservation (sched_dump, insn); | |
6455 | } | |
6456 | fprintf (sched_dump, "\n"); | |
6457 | } | |
6458 | } | |
6459 | ||
496d7bb0 MK |
6460 | /* Use forward list scheduling to rearrange insns of block pointed to by |
6461 | TARGET_BB, possibly bringing insns from subsequent blocks in the same | |
6462 | region. */ | |
8c660648 | 6463 | |
06d7e8e7 | 6464 | bool |
975ccf22 | 6465 | schedule_block (basic_block *target_bb, state_t init_state) |
8c660648 | 6466 | { |
894fd6f2 | 6467 | int i; |
06d7e8e7 | 6468 | bool success = modulo_ii == 0; |
51564684 | 6469 | struct sched_block_state ls; |
fae15c93 | 6470 | state_t temp_state = NULL; /* It is used for multipass scheduling. */ |
58fb7809 | 6471 | int sort_p, advance, start_clock_var; |
8c660648 | 6472 | |
63de6c74 | 6473 | /* Head/tail info for this block. */ |
ce1ce33a | 6474 | rtx_insn *prev_head = current_sched_info->prev_head; |
dc01c3d1 | 6475 | rtx_insn *next_tail = current_sched_info->next_tail; |
ce1ce33a DM |
6476 | rtx_insn *head = NEXT_INSN (prev_head); |
6477 | rtx_insn *tail = PREV_INSN (next_tail); | |
8c660648 | 6478 | |
4f66c9bc | 6479 | if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0 |
b16abbcb | 6480 | && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion) |
1a83e602 BS |
6481 | find_modifiable_mems (head, tail); |
6482 | ||
484df988 JL |
6483 | /* We used to have code to avoid getting parameters moved from hard |
6484 | argument registers into pseudos. | |
8c660648 | 6485 | |
484df988 JL |
6486 | However, it was removed when it proved to be of marginal benefit |
6487 | and caused problems because schedule_block and compute_forward_dependences | |
6488 | had different notions of what the "head" insn was. */ | |
8c660648 | 6489 | |
535a42b1 | 6490 | gcc_assert (head != tail || INSN_P (head)); |
8c660648 | 6491 | |
e2f6ff94 | 6492 | haifa_recovery_bb_recently_added_p = false; |
496d7bb0 | 6493 | |
26965010 BS |
6494 | backtrack_queue = NULL; |
6495 | ||
63de6c74 | 6496 | /* Debug info. */ |
8c660648 | 6497 | if (sched_verbose) |
d5c67efd MK |
6498 | { |
6499 | dump_new_block_header (0, *target_bb, head, tail); | |
6500 | ||
6501 | if (sched_verbose >= 2) | |
4bd12f3d MK |
6502 | { |
6503 | dump_insn_stream (head, tail); | |
6504 | memset (&rank_for_schedule_stats, 0, | |
6505 | sizeof (rank_for_schedule_stats)); | |
6506 | } | |
d5c67efd | 6507 | } |
8c660648 | 6508 | |
975ccf22 BS |
6509 | if (init_state == NULL) |
6510 | state_reset (curr_state); | |
6511 | else | |
6512 | memcpy (curr_state, init_state, dfa_state_size); | |
8c660648 | 6513 | |
e855c69d | 6514 | /* Clear the ready list. */ |
176f9a7b | 6515 | ready.first = ready.veclen - 1; |
176f9a7b | 6516 | ready.n_ready = 0; |
b5b8b0ac | 6517 | ready.n_debug = 0; |
8c660648 | 6518 | |
fa0aee89 PB |
6519 | /* It is used for first cycle multipass scheduling. */ |
6520 | temp_state = alloca (dfa_state_size); | |
fae15c93 | 6521 | |
38f8b050 JR |
6522 | if (targetm.sched.init) |
6523 | targetm.sched.init (sched_dump, sched_verbose, ready.veclen); | |
e4da5f6d | 6524 | |
89076bb3 | 6525 | /* We start inserting insns after PREV_HEAD. */ |
06bc63d1 | 6526 | last_scheduled_insn = prev_head; |
90831096 | 6527 | last_nondebug_scheduled_insn = NULL; |
ce1ce33a | 6528 | nonscheduled_insns_begin = NULL; |
8c660648 | 6529 | |
b5b8b0ac | 6530 | gcc_assert ((NOTE_P (last_scheduled_insn) |
a59d15cf | 6531 | || DEBUG_INSN_P (last_scheduled_insn)) |
496d7bb0 MK |
6532 | && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb); |
6533 | ||
1708fd40 BS |
6534 | /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the |
6535 | queue. */ | |
8c660648 JL |
6536 | q_ptr = 0; |
6537 | q_size = 0; | |
fae15c93 | 6538 | |
3dc99c19 | 6539 | insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1); |
fa0aee89 | 6540 | memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx)); |
8c660648 | 6541 | |
197043f5 RH |
6542 | /* Start just before the beginning of time. */ |
6543 | clock_var = -1; | |
63f54b1a | 6544 | |
b8698a0f | 6545 | /* We need queue and ready lists and clock_var be initialized |
63f54b1a MK |
6546 | in try_ready () (which is called through init_ready_list ()). */ |
6547 | (*current_sched_info->init_ready_list) (); | |
6548 | ||
4dd9ac6c MK |
6549 | if (sched_pressure) |
6550 | sched_pressure_start_bb (*target_bb); | |
7a1aca9c | 6551 | |
6f8dd94b EB |
6552 | /* The algorithm is O(n^2) in the number of ready insns at any given |
6553 | time in the worst case. Before reload we are more likely to have | |
6554 | big lists so truncate them to a reasonable size. */ | |
b5b8b0ac AO |
6555 | if (!reload_completed |
6556 | && ready.n_ready - ready.n_debug > MAX_SCHED_READY_INSNS) | |
6f8dd94b | 6557 | { |
696d5fa1 MK |
6558 | ready_sort_debug (&ready); |
6559 | ready_sort_real (&ready); | |
6f8dd94b | 6560 | |
b5b8b0ac AO |
6561 | /* Find first free-standing insn past MAX_SCHED_READY_INSNS. |
6562 | If there are debug insns, we know they're first. */ | |
6563 | for (i = MAX_SCHED_READY_INSNS + ready.n_debug; i < ready.n_ready; i++) | |
6f8dd94b EB |
6564 | if (!SCHED_GROUP_P (ready_element (&ready, i))) |
6565 | break; | |
6566 | ||
6567 | if (sched_verbose >= 2) | |
6568 | { | |
6569 | fprintf (sched_dump, | |
696d5fa1 MK |
6570 | ";;\t\tReady list on entry: %d insns: ", ready.n_ready); |
6571 | debug_ready_list (&ready); | |
6f8dd94b EB |
6572 | fprintf (sched_dump, |
6573 | ";;\t\t before reload => truncated to %d insns\n", i); | |
6574 | } | |
6575 | ||
b631c5f7 SP |
6576 | /* Delay all insns past it for 1 cycle. If debug counter is |
6577 | activated make an exception for the insn right after | |
9b69cf83 | 6578 | nonscheduled_insns_begin. */ |
b631c5f7 | 6579 | { |
ce1ce33a | 6580 | rtx_insn *skip_insn; |
b631c5f7 SP |
6581 | |
6582 | if (dbg_cnt (sched_insn) == false) | |
06bc63d1 | 6583 | skip_insn = first_nonscheduled_insn (); |
b631c5f7 | 6584 | else |
ce1ce33a | 6585 | skip_insn = NULL; |
b631c5f7 SP |
6586 | |
6587 | while (i < ready.n_ready) | |
6588 | { | |
ce1ce33a | 6589 | rtx_insn *insn; |
b631c5f7 SP |
6590 | |
6591 | insn = ready_remove (&ready, i); | |
6592 | ||
6593 | if (insn != skip_insn) | |
ec4efea9 | 6594 | queue_insn (insn, 1, "list truncated"); |
b631c5f7 | 6595 | } |
9b69cf83 BS |
6596 | if (skip_insn) |
6597 | ready_add (&ready, skip_insn, true); | |
b631c5f7 | 6598 | } |
6f8dd94b EB |
6599 | } |
6600 | ||
496d7bb0 MK |
6601 | /* Now we can restore basic block notes and maintain precise cfg. */ |
6602 | restore_bb_notes (*target_bb); | |
6603 | ||
63f54b1a MK |
6604 | last_clock_var = -1; |
6605 | ||
58fb7809 | 6606 | advance = 0; |
197043f5 | 6607 | |
9771b263 | 6608 | gcc_assert (scheduled_insns.length () == 0); |
30028c85 | 6609 | sort_p = TRUE; |
26965010 | 6610 | must_backtrack = false; |
06d7e8e7 BS |
6611 | modulo_insns_scheduled = 0; |
6612 | ||
6613 | ls.modulo_epilogue = false; | |
f50d71ec | 6614 | ls.first_cycle_insn_p = true; |
26965010 | 6615 | |
63de6c74 | 6616 | /* Loop until all the insns in BB are scheduled. */ |
1708fd40 | 6617 | while ((*current_sched_info->schedule_more_p) ()) |
8c660648 | 6618 | { |
1a83e602 | 6619 | perform_replacements_new_cycle (); |
58fb7809 | 6620 | do |
8c660648 | 6621 | { |
58fb7809 VM |
6622 | start_clock_var = clock_var; |
6623 | ||
6624 | clock_var++; | |
1d088dee | 6625 | |
58fb7809 | 6626 | advance_one_cycle (); |
1d088dee | 6627 | |
58fb7809 VM |
6628 | /* Add to the ready list all pending insns that can be issued now. |
6629 | If there are no ready insns, increment clock until one | |
6630 | is ready and add all pending insns at that point to the ready | |
6631 | list. */ | |
6632 | queue_to_ready (&ready); | |
1d088dee | 6633 | |
535a42b1 | 6634 | gcc_assert (ready.n_ready); |
1d088dee | 6635 | |
58fb7809 VM |
6636 | if (sched_verbose >= 2) |
6637 | { | |
d5c67efd | 6638 | fprintf (sched_dump, ";;\t\tReady list after queue_to_ready:"); |
58fb7809 VM |
6639 | debug_ready_list (&ready); |
6640 | } | |
6641 | advance -= clock_var - start_clock_var; | |
8c660648 | 6642 | } |
58fb7809 | 6643 | while (advance > 0); |
8c660648 | 6644 | |
06d7e8e7 BS |
6645 | if (ls.modulo_epilogue) |
6646 | { | |
6647 | int stage = clock_var / modulo_ii; | |
6648 | if (stage > modulo_last_stage * 2 + 2) | |
6649 | { | |
6650 | if (sched_verbose >= 2) | |
6651 | fprintf (sched_dump, | |
6652 | ";;\t\tmodulo scheduled succeeded at II %d\n", | |
6653 | modulo_ii); | |
6654 | success = true; | |
6655 | goto end_schedule; | |
6656 | } | |
6657 | } | |
6658 | else if (modulo_ii > 0) | |
6659 | { | |
6660 | int stage = clock_var / modulo_ii; | |
6661 | if (stage > modulo_max_stages) | |
6662 | { | |
6663 | if (sched_verbose >= 2) | |
6664 | fprintf (sched_dump, | |
6665 | ";;\t\tfailing schedule due to excessive stages\n"); | |
6666 | goto end_schedule; | |
6667 | } | |
6668 | if (modulo_n_insns == modulo_insns_scheduled | |
6669 | && stage > modulo_last_stage) | |
6670 | { | |
6671 | if (sched_verbose >= 2) | |
6672 | fprintf (sched_dump, | |
6673 | ";;\t\tfound kernel after %d stages, II %d\n", | |
6674 | stage, modulo_ii); | |
6675 | ls.modulo_epilogue = true; | |
6676 | } | |
6677 | } | |
6678 | ||
6679 | prune_ready_list (temp_state, true, false, ls.modulo_epilogue); | |
9d701248 | 6680 | if (ready.n_ready == 0) |
7b1ac803 | 6681 | continue; |
26965010 BS |
6682 | if (must_backtrack) |
6683 | goto do_backtrack; | |
9d701248 | 6684 | |
26965010 | 6685 | ls.shadows_only_p = false; |
7b1ac803 | 6686 | cycle_issued_insns = 0; |
51564684 | 6687 | ls.can_issue_more = issue_rate; |
7b1ac803 | 6688 | for (;;) |
30028c85 | 6689 | { |
ce1ce33a | 6690 | rtx_insn *insn; |
7b1ac803 | 6691 | int cost; |
26965010 | 6692 | bool asm_p; |
1d088dee | 6693 | |
7b1ac803 | 6694 | if (sort_p && ready.n_ready > 0) |
30028c85 | 6695 | { |
7b1ac803 BS |
6696 | /* Sort the ready list based on priority. This must be |
6697 | done every iteration through the loop, as schedule_insn | |
6698 | may have readied additional insns that will not be | |
6699 | sorted correctly. */ | |
6700 | ready_sort (&ready); | |
6701 | ||
6702 | if (sched_verbose >= 2) | |
6703 | { | |
d5c67efd MK |
6704 | fprintf (sched_dump, |
6705 | ";;\t\tReady list after ready_sort: "); | |
7b1ac803 BS |
6706 | debug_ready_list (&ready); |
6707 | } | |
30028c85 | 6708 | } |
197043f5 | 6709 | |
7b1ac803 BS |
6710 | /* We don't want md sched reorder to even see debug isns, so put |
6711 | them out right away. */ | |
6712 | if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)) | |
6713 | && (*current_sched_info->schedule_more_p) ()) | |
b5b8b0ac | 6714 | { |
7b1ac803 BS |
6715 | while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))) |
6716 | { | |
ce1ce33a | 6717 | rtx_insn *insn = ready_remove_first (&ready); |
7b1ac803 BS |
6718 | gcc_assert (DEBUG_INSN_P (insn)); |
6719 | (*current_sched_info->begin_schedule_ready) (insn); | |
9771b263 | 6720 | scheduled_insns.safe_push (insn); |
7b1ac803 BS |
6721 | last_scheduled_insn = insn; |
6722 | advance = schedule_insn (insn); | |
6723 | gcc_assert (advance == 0); | |
6724 | if (ready.n_ready > 0) | |
6725 | ready_sort (&ready); | |
6726 | } | |
b5b8b0ac AO |
6727 | } |
6728 | ||
51564684 | 6729 | if (ls.first_cycle_insn_p && !ready.n_ready) |
7b1ac803 | 6730 | break; |
e1306f49 | 6731 | |
26965010 | 6732 | resume_after_backtrack: |
7b1ac803 BS |
6733 | /* Allow the target to reorder the list, typically for |
6734 | better instruction bundling. */ | |
6735 | if (sort_p | |
6736 | && (ready.n_ready == 0 | |
6737 | || !SCHED_GROUP_P (ready_element (&ready, 0)))) | |
6738 | { | |
51564684 BS |
6739 | if (ls.first_cycle_insn_p && targetm.sched.reorder) |
6740 | ls.can_issue_more | |
7b1ac803 BS |
6741 | = targetm.sched.reorder (sched_dump, sched_verbose, |
6742 | ready_lastpos (&ready), | |
6743 | &ready.n_ready, clock_var); | |
51564684 BS |
6744 | else if (!ls.first_cycle_insn_p && targetm.sched.reorder2) |
6745 | ls.can_issue_more | |
7b1ac803 BS |
6746 | = targetm.sched.reorder2 (sched_dump, sched_verbose, |
6747 | ready.n_ready | |
6748 | ? ready_lastpos (&ready) : NULL, | |
6749 | &ready.n_ready, clock_var); | |
6750 | } | |
fae15c93 | 6751 | |
7b1ac803 | 6752 | restart_choose_ready: |
6d0de005 | 6753 | if (sched_verbose >= 2) |
fae15c93 | 6754 | { |
496d7bb0 | 6755 | fprintf (sched_dump, ";;\tReady list (t = %3d): ", |
fae15c93 VM |
6756 | clock_var); |
6757 | debug_ready_list (&ready); | |
60867e8c | 6758 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) |
ce18efcb | 6759 | print_curr_reg_pressure (); |
fae15c93 VM |
6760 | } |
6761 | ||
b8698a0f | 6762 | if (ready.n_ready == 0 |
51564684 | 6763 | && ls.can_issue_more |
b8698a0f | 6764 | && reload_completed) |
fae15c93 | 6765 | { |
fa0aee89 PB |
6766 | /* Allow scheduling insns directly from the queue in case |
6767 | there's nothing better to do (ready list is empty) but | |
6768 | there are still vacant dispatch slots in the current cycle. */ | |
6769 | if (sched_verbose >= 6) | |
62e5bf5d | 6770 | fprintf (sched_dump,";;\t\tSecond chance\n"); |
fa0aee89 PB |
6771 | memcpy (temp_state, curr_state, dfa_state_size); |
6772 | if (early_queue_to_ready (temp_state, &ready)) | |
6773 | ready_sort (&ready); | |
fae15c93 | 6774 | } |
569fa502 | 6775 | |
b5b8b0ac | 6776 | if (ready.n_ready == 0 |
51564684 | 6777 | || !ls.can_issue_more |
fa0aee89 PB |
6778 | || state_dead_lock_p (curr_state) |
6779 | || !(*current_sched_info->schedule_more_p) ()) | |
6780 | break; | |
1d088dee | 6781 | |
fa0aee89 PB |
6782 | /* Select and remove the insn from the ready list. */ |
6783 | if (sort_p) | |
496d7bb0 | 6784 | { |
b631c5f7 SP |
6785 | int res; |
6786 | ||
ce1ce33a | 6787 | insn = NULL; |
51564684 | 6788 | res = choose_ready (&ready, ls.first_cycle_insn_p, &insn); |
b631c5f7 SP |
6789 | |
6790 | if (res < 0) | |
6791 | /* Finish cycle. */ | |
6792 | break; | |
6793 | if (res > 0) | |
7b1ac803 | 6794 | goto restart_choose_ready; |
b631c5f7 SP |
6795 | |
6796 | gcc_assert (insn != NULL_RTX); | |
496d7bb0 | 6797 | } |
fa0aee89 PB |
6798 | else |
6799 | insn = ready_remove_first (&ready); | |
1d088dee | 6800 | |
7a1aca9c | 6801 | if (sched_pressure != SCHED_PRESSURE_NONE |
60867e8c | 6802 | && INSN_TICK (insn) > clock_var) |
ce18efcb VM |
6803 | { |
6804 | ready_add (&ready, insn, true); | |
6805 | advance = 1; | |
6806 | break; | |
6807 | } | |
6808 | ||
fa0aee89 PB |
6809 | if (targetm.sched.dfa_new_cycle |
6810 | && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, | |
6811 | insn, last_clock_var, | |
6812 | clock_var, &sort_p)) | |
63f54b1a MK |
6813 | /* SORT_P is used by the target to override sorting |
6814 | of the ready list. This is needed when the target | |
6815 | has modified its internal structures expecting that | |
6816 | the insn will be issued next. As we need the insn | |
6817 | to have the highest priority (so it will be returned by | |
6818 | the ready_remove_first call above), we invoke | |
6819 | ready_add (&ready, insn, true). | |
b8698a0f L |
6820 | But, still, there is one issue: INSN can be later |
6821 | discarded by scheduler's front end through | |
63f54b1a | 6822 | current_sched_info->can_schedule_ready_p, hence, won't |
b8698a0f | 6823 | be issued next. */ |
fa0aee89 | 6824 | { |
63f54b1a MK |
6825 | ready_add (&ready, insn, true); |
6826 | break; | |
fa0aee89 | 6827 | } |
1d088dee | 6828 | |
fa0aee89 | 6829 | sort_p = TRUE; |
e1306f49 | 6830 | |
496d7bb0 MK |
6831 | if (current_sched_info->can_schedule_ready_p |
6832 | && ! (*current_sched_info->can_schedule_ready_p) (insn)) | |
6833 | /* We normally get here only if we don't want to move | |
6834 | insn from the split block. */ | |
6835 | { | |
1a83e602 | 6836 | TODO_SPEC (insn) = DEP_POSTPONED; |
7b1ac803 | 6837 | goto restart_choose_ready; |
496d7bb0 MK |
6838 | } |
6839 | ||
c203e8a7 | 6840 | if (delay_htab) |
26965010 BS |
6841 | { |
6842 | /* If this insn is the first part of a delay-slot pair, record a | |
6843 | backtrack point. */ | |
6844 | struct delay_pair *delay_entry; | |
6845 | delay_entry | |
c203e8a7 | 6846 | = delay_htab->find_with_hash (insn, htab_hash_pointer (insn)); |
26965010 BS |
6847 | if (delay_entry) |
6848 | { | |
6849 | save_backtrack_point (delay_entry, ls); | |
6850 | if (sched_verbose >= 2) | |
6851 | fprintf (sched_dump, ";;\t\tsaving backtrack point\n"); | |
6852 | } | |
6853 | } | |
6854 | ||
b8698a0f L |
6855 | /* DECISION is made. */ |
6856 | ||
06d7e8e7 BS |
6857 | if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid) |
6858 | { | |
6859 | modulo_insns_scheduled++; | |
6860 | modulo_last_stage = clock_var / modulo_ii; | |
6861 | } | |
496d7bb0 MK |
6862 | if (TODO_SPEC (insn) & SPECULATIVE) |
6863 | generate_recovery_code (insn); | |
6864 | ||
ac44248e | 6865 | if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON)) |
7942e47e RY |
6866 | targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW); |
6867 | ||
86014d07 BS |
6868 | /* Update counters, etc in the scheduler's front end. */ |
6869 | (*current_sched_info->begin_schedule_ready) (insn); | |
9771b263 | 6870 | scheduled_insns.safe_push (insn); |
5c24671b AO |
6871 | gcc_assert (NONDEBUG_INSN_P (insn)); |
6872 | last_nondebug_scheduled_insn = last_scheduled_insn = insn; | |
b8698a0f | 6873 | |
9d701248 BS |
6874 | if (recog_memoized (insn) >= 0) |
6875 | { | |
fca96842 | 6876 | memcpy (temp_state, curr_state, dfa_state_size); |
9d701248 | 6877 | cost = state_transition (curr_state, insn); |
b16abbcb | 6878 | if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion) |
9d701248 | 6879 | gcc_assert (cost < 0); |
fca96842 BS |
6880 | if (memcmp (temp_state, curr_state, dfa_state_size) != 0) |
6881 | cycle_issued_insns++; | |
9d701248 BS |
6882 | asm_p = false; |
6883 | } | |
6884 | else | |
6885 | asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT | |
6886 | || asm_noperands (PATTERN (insn)) >= 0); | |
1d088dee | 6887 | |
c237e94a | 6888 | if (targetm.sched.variable_issue) |
51564684 | 6889 | ls.can_issue_more = |
5fd9b178 | 6890 | targetm.sched.variable_issue (sched_dump, sched_verbose, |
51564684 | 6891 | insn, ls.can_issue_more); |
85d69216 JL |
6892 | /* A naked CLOBBER or USE generates no instruction, so do |
6893 | not count them against the issue rate. */ | |
6894 | else if (GET_CODE (PATTERN (insn)) != USE | |
6895 | && GET_CODE (PATTERN (insn)) != CLOBBER) | |
51564684 | 6896 | ls.can_issue_more--; |
63f54b1a | 6897 | advance = schedule_insn (insn); |
d57f1617 | 6898 | |
26965010 BS |
6899 | if (SHADOW_P (insn)) |
6900 | ls.shadows_only_p = true; | |
6901 | ||
d57f1617 VM |
6902 | /* After issuing an asm insn we should start a new cycle. */ |
6903 | if (advance == 0 && asm_p) | |
6904 | advance = 1; | |
26965010 BS |
6905 | |
6906 | if (must_backtrack) | |
6907 | break; | |
6908 | ||
58fb7809 VM |
6909 | if (advance != 0) |
6910 | break; | |
8c660648 | 6911 | |
51564684 | 6912 | ls.first_cycle_insn_p = false; |
30028c85 | 6913 | if (ready.n_ready > 0) |
06d7e8e7 BS |
6914 | prune_ready_list (temp_state, false, ls.shadows_only_p, |
6915 | ls.modulo_epilogue); | |
b4ead7d4 | 6916 | } |
8c660648 | 6917 | |
26965010 BS |
6918 | do_backtrack: |
6919 | if (!must_backtrack) | |
6920 | for (i = 0; i < ready.n_ready; i++) | |
6921 | { | |
f57aa6b0 | 6922 | rtx_insn *insn = ready_element (&ready, i); |
26965010 BS |
6923 | if (INSN_EXACT_TICK (insn) == clock_var) |
6924 | { | |
6925 | must_backtrack = true; | |
6926 | clock_var++; | |
6927 | break; | |
6928 | } | |
6929 | } | |
06d7e8e7 BS |
6930 | if (must_backtrack && modulo_ii > 0) |
6931 | { | |
6932 | if (modulo_backtracks_left == 0) | |
6933 | goto end_schedule; | |
6934 | modulo_backtracks_left--; | |
6935 | } | |
26965010 BS |
6936 | while (must_backtrack) |
6937 | { | |
6938 | struct haifa_saved_data *failed; | |
ce1ce33a | 6939 | rtx_insn *failed_insn; |
26965010 BS |
6940 | |
6941 | must_backtrack = false; | |
6942 | failed = verify_shadows (); | |
6943 | gcc_assert (failed); | |
6944 | ||
6945 | failed_insn = failed->delay_pair->i1; | |
1a83e602 BS |
6946 | /* Clear these queues. */ |
6947 | perform_replacements_new_cycle (); | |
e2724e63 | 6948 | toggle_cancelled_flags (false); |
26965010 BS |
6949 | unschedule_insns_until (failed_insn); |
6950 | while (failed != backtrack_queue) | |
6951 | free_topmost_backtrack_point (true); | |
6952 | restore_last_backtrack_point (&ls); | |
6953 | if (sched_verbose >= 2) | |
6954 | fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var); | |
6955 | /* Delay by at least a cycle. This could cause additional | |
6956 | backtracking. */ | |
6957 | queue_insn (failed_insn, 1, "backtracked"); | |
6958 | advance = 0; | |
6959 | if (must_backtrack) | |
6960 | continue; | |
6961 | if (ready.n_ready > 0) | |
6962 | goto resume_after_backtrack; | |
6963 | else | |
6964 | { | |
6965 | if (clock_var == 0 && ls.first_cycle_insn_p) | |
6966 | goto end_schedule; | |
6967 | advance = 1; | |
6968 | break; | |
6969 | } | |
6970 | } | |
f50d71ec | 6971 | ls.first_cycle_insn_p = true; |
26965010 | 6972 | } |
06d7e8e7 BS |
6973 | if (ls.modulo_epilogue) |
6974 | success = true; | |
26965010 | 6975 | end_schedule: |
2fb5f0da | 6976 | if (!ls.first_cycle_insn_p || advance) |
f50d71ec | 6977 | advance_one_cycle (); |
1a83e602 | 6978 | perform_replacements_new_cycle (); |
06d7e8e7 BS |
6979 | if (modulo_ii > 0) |
6980 | { | |
6981 | /* Once again, debug insn suckiness: they can be on the ready list | |
6982 | even if they have unresolved dependencies. To make our view | |
6983 | of the world consistent, remove such "ready" insns. */ | |
6984 | restart_debug_insn_loop: | |
6985 | for (i = ready.n_ready - 1; i >= 0; i--) | |
6986 | { | |
f57aa6b0 | 6987 | rtx_insn *x; |
06d7e8e7 BS |
6988 | |
6989 | x = ready_element (&ready, i); | |
6990 | if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL | |
6991 | || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL) | |
6992 | { | |
6993 | ready_remove (&ready, i); | |
6994 | goto restart_debug_insn_loop; | |
6995 | } | |
6996 | } | |
6997 | for (i = ready.n_ready - 1; i >= 0; i--) | |
6998 | { | |
ce1ce33a | 6999 | rtx_insn *x; |
06d7e8e7 BS |
7000 | |
7001 | x = ready_element (&ready, i); | |
7002 | resolve_dependencies (x); | |
7003 | } | |
7004 | for (i = 0; i <= max_insn_queue_index; i++) | |
7005 | { | |
3dc99c19 | 7006 | rtx_insn_list *link; |
06d7e8e7 BS |
7007 | while ((link = insn_queue[i]) != NULL) |
7008 | { | |
3dc99c19 DM |
7009 | rtx_insn *x = link->insn (); |
7010 | insn_queue[i] = link->next (); | |
06d7e8e7 BS |
7011 | QUEUE_INDEX (x) = QUEUE_NOWHERE; |
7012 | free_INSN_LIST_node (link); | |
7013 | resolve_dependencies (x); | |
7014 | } | |
7015 | } | |
7016 | } | |
7017 | ||
1a83e602 BS |
7018 | if (!success) |
7019 | undo_all_replacements (); | |
7020 | ||
b4ead7d4 BS |
7021 | /* Debug info. */ |
7022 | if (sched_verbose) | |
7023 | { | |
7024 | fprintf (sched_dump, ";;\tReady list (final): "); | |
7025 | debug_ready_list (&ready); | |
b4ead7d4 | 7026 | } |
8c660648 | 7027 | |
06d7e8e7 | 7028 | if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty) |
63f54b1a MK |
7029 | /* Sanity check -- queue must be empty now. Meaningless if region has |
7030 | multiple bbs. */ | |
b5b8b0ac | 7031 | gcc_assert (!q_size && !ready.n_ready && !ready.n_debug); |
06d7e8e7 | 7032 | else if (modulo_ii == 0) |
30028c85 | 7033 | { |
63f54b1a MK |
7034 | /* We must maintain QUEUE_INDEX between blocks in region. */ |
7035 | for (i = ready.n_ready - 1; i >= 0; i--) | |
496d7bb0 | 7036 | { |
f57aa6b0 | 7037 | rtx_insn *x; |
b8698a0f | 7038 | |
496d7bb0 MK |
7039 | x = ready_element (&ready, i); |
7040 | QUEUE_INDEX (x) = QUEUE_NOWHERE; | |
b953c2b8 | 7041 | TODO_SPEC (x) = HARD_DEP; |
496d7bb0 | 7042 | } |
63f54b1a | 7043 | |
b8698a0f | 7044 | if (q_size) |
63f54b1a MK |
7045 | for (i = 0; i <= max_insn_queue_index; i++) |
7046 | { | |
96b70414 DM |
7047 | rtx_insn_list *link; |
7048 | for (link = insn_queue[i]; link; link = link->next ()) | |
496d7bb0 | 7049 | { |
f57aa6b0 | 7050 | rtx_insn *x; |
496d7bb0 | 7051 | |
96b70414 | 7052 | x = link->insn (); |
496d7bb0 | 7053 | QUEUE_INDEX (x) = QUEUE_NOWHERE; |
b953c2b8 | 7054 | TODO_SPEC (x) = HARD_DEP; |
496d7bb0 | 7055 | } |
63f54b1a MK |
7056 | free_INSN_LIST_list (&insn_queue[i]); |
7057 | } | |
496d7bb0 | 7058 | } |
1d088dee | 7059 | |
7a1aca9c RS |
7060 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
7061 | model_end_schedule (); | |
7062 | ||
06d7e8e7 BS |
7063 | if (success) |
7064 | { | |
7065 | commit_schedule (prev_head, tail, target_bb); | |
7066 | if (sched_verbose) | |
7067 | fprintf (sched_dump, ";; total time = %d\n", clock_var); | |
7068 | } | |
7069 | else | |
7070 | last_scheduled_insn = tail; | |
7071 | ||
9771b263 | 7072 | scheduled_insns.truncate (0); |
e855c69d | 7073 | |
496d7bb0 | 7074 | if (!current_sched_info->queue_must_finish_empty |
e2f6ff94 | 7075 | || haifa_recovery_bb_recently_added_p) |
496d7bb0 | 7076 | { |
30028c85 VM |
7077 | /* INSN_TICK (minimum clock tick at which the insn becomes |
7078 | ready) may be not correct for the insn in the subsequent | |
7079 | blocks of the region. We should use a correct value of | |
7080 | `clock_var' or modify INSN_TICK. It is better to keep | |
7081 | clock_var value equal to 0 at the start of a basic block. | |
7082 | Therefore we modify INSN_TICK here. */ | |
63f54b1a | 7083 | fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn); |
30028c85 VM |
7084 | } |
7085 | ||
38f8b050 | 7086 | if (targetm.sched.finish) |
e2f6ff94 | 7087 | { |
38f8b050 | 7088 | targetm.sched.finish (sched_dump, sched_verbose); |
fa10beec | 7089 | /* Target might have added some instructions to the scheduled block |
e2f6ff94 MK |
7090 | in its md_finish () hook. These new insns don't have any data |
7091 | initialized and to identify them we extend h_i_d so that they'll | |
e855c69d | 7092 | get zero luids. */ |
a95b23b4 | 7093 | sched_extend_luids (); |
e2f6ff94 | 7094 | } |
63f54b1a MK |
7095 | |
7096 | /* Update head/tail boundaries. */ | |
7097 | head = NEXT_INSN (prev_head); | |
7098 | tail = last_scheduled_insn; | |
7099 | ||
d5c67efd MK |
7100 | if (sched_verbose) |
7101 | { | |
7102 | fprintf (sched_dump, ";; new head = %d\n;; new tail = %d\n", | |
7103 | INSN_UID (head), INSN_UID (tail)); | |
7104 | ||
7105 | if (sched_verbose >= 2) | |
4bd12f3d MK |
7106 | { |
7107 | dump_insn_stream (head, tail); | |
b75ae7f4 MK |
7108 | print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats, |
7109 | NULL); | |
4bd12f3d | 7110 | } |
d5c67efd MK |
7111 | |
7112 | fprintf (sched_dump, "\n"); | |
7113 | } | |
7114 | ||
e855c69d | 7115 | head = restore_other_notes (head, NULL); |
8c660648 | 7116 | |
b4ead7d4 BS |
7117 | current_sched_info->head = head; |
7118 | current_sched_info->tail = tail; | |
26965010 BS |
7119 | |
7120 | free_backtrack_queue (); | |
06d7e8e7 BS |
7121 | |
7122 | return success; | |
8c660648 | 7123 | } |
b4ead7d4 | 7124 | \f |
63de6c74 | 7125 | /* Set_priorities: compute priority of each insn in the block. */ |
8c660648 | 7126 | |
b4ead7d4 | 7127 | int |
0e0ce50d | 7128 | set_priorities (rtx_insn *head, rtx_insn *tail) |
8c660648 | 7129 | { |
0e0ce50d | 7130 | rtx_insn *insn; |
8c660648 | 7131 | int n_insn; |
b8698a0f | 7132 | int sched_max_insns_priority = |
79ae11c4 | 7133 | current_sched_info->sched_max_insns_priority; |
ce1ce33a | 7134 | rtx_insn *prev_head; |
8c660648 | 7135 | |
a59d15cf | 7136 | if (head == tail && ! INSN_P (head)) |
b5b8b0ac | 7137 | gcc_unreachable (); |
8c660648 JL |
7138 | |
7139 | n_insn = 0; | |
63f54b1a MK |
7140 | |
7141 | prev_head = PREV_INSN (head); | |
8c660648 JL |
7142 | for (insn = tail; insn != prev_head; insn = PREV_INSN (insn)) |
7143 | { | |
63f54b1a | 7144 | if (!INSN_P (insn)) |
8c660648 JL |
7145 | continue; |
7146 | ||
58fb7809 | 7147 | n_insn++; |
8c660648 | 7148 | (void) priority (insn); |
79ae11c4 | 7149 | |
916fa4f0 MK |
7150 | gcc_assert (INSN_PRIORITY_KNOWN (insn)); |
7151 | ||
7152 | sched_max_insns_priority = MAX (sched_max_insns_priority, | |
7153 | INSN_PRIORITY (insn)); | |
8c660648 | 7154 | } |
63f54b1a MK |
7155 | |
7156 | current_sched_info->sched_max_insns_priority = sched_max_insns_priority; | |
8c660648 JL |
7157 | |
7158 | return n_insn; | |
7159 | } | |
7160 | ||
4111f1c9 | 7161 | /* Set sched_dump and sched_verbose for the desired debugging output. */ |
e855c69d AB |
7162 | void |
7163 | setup_sched_dump (void) | |
7164 | { | |
7165 | sched_verbose = sched_verbose_param; | |
4111f1c9 NB |
7166 | sched_dump = dump_file; |
7167 | if (!dump_file) | |
7168 | sched_verbose = 0; | |
e855c69d | 7169 | } |
496d7bb0 | 7170 | |
c4cd7435 AB |
7171 | /* Allocate data for register pressure sensitive scheduling. */ |
7172 | static void | |
7173 | alloc_global_sched_pressure_data (void) | |
7174 | { | |
7175 | if (sched_pressure != SCHED_PRESSURE_NONE) | |
7176 | { | |
7177 | int i, max_regno = max_reg_num (); | |
7178 | ||
7179 | if (sched_dump != NULL) | |
7180 | /* We need info about pseudos for rtl dumps about pseudo | |
7181 | classes and costs. */ | |
7182 | regstat_init_n_sets_and_refs (); | |
7183 | ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL); | |
7184 | sched_regno_pressure_class | |
7185 | = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class)); | |
7186 | for (i = 0; i < max_regno; i++) | |
7187 | sched_regno_pressure_class[i] | |
7188 | = (i < FIRST_PSEUDO_REGISTER | |
7189 | ? ira_pressure_class_translate[REGNO_REG_CLASS (i)] | |
7190 | : ira_pressure_class_translate[reg_allocno_class (i)]); | |
7191 | curr_reg_live = BITMAP_ALLOC (NULL); | |
7192 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) | |
7193 | { | |
7194 | saved_reg_live = BITMAP_ALLOC (NULL); | |
7195 | region_ref_regs = BITMAP_ALLOC (NULL); | |
7196 | } | |
f380f608 RS |
7197 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
7198 | tmp_bitmap = BITMAP_ALLOC (NULL); | |
4dd9ac6c | 7199 | |
6d5c37d0 PH |
7200 | /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes |
7201 | that we calculate register pressure for. */ | |
4dd9ac6c MK |
7202 | for (int c = 0; c < ira_pressure_classes_num; ++c) |
7203 | { | |
7204 | enum reg_class cl = ira_pressure_classes[c]; | |
7205 | ||
6d5c37d0 PH |
7206 | call_saved_regs_num[cl] = 0; |
7207 | fixed_regs_num[cl] = 0; | |
4dd9ac6c MK |
7208 | |
7209 | for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i) | |
6d5c37d0 PH |
7210 | if (!call_used_regs[ira_class_hard_regs[cl][i]]) |
7211 | ++call_saved_regs_num[cl]; | |
7212 | else if (fixed_regs[ira_class_hard_regs[cl][i]]) | |
7213 | ++fixed_regs_num[cl]; | |
4dd9ac6c | 7214 | } |
c4cd7435 AB |
7215 | } |
7216 | } | |
7217 | ||
7218 | /* Free data for register pressure sensitive scheduling. Also called | |
7219 | from schedule_region when stopping sched-pressure early. */ | |
7220 | void | |
7221 | free_global_sched_pressure_data (void) | |
7222 | { | |
7223 | if (sched_pressure != SCHED_PRESSURE_NONE) | |
7224 | { | |
7225 | if (regstat_n_sets_and_refs != NULL) | |
7226 | regstat_free_n_sets_and_refs (); | |
7227 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) | |
7228 | { | |
7229 | BITMAP_FREE (region_ref_regs); | |
7230 | BITMAP_FREE (saved_reg_live); | |
7231 | } | |
f380f608 RS |
7232 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
7233 | BITMAP_FREE (tmp_bitmap); | |
c4cd7435 AB |
7234 | BITMAP_FREE (curr_reg_live); |
7235 | free (sched_regno_pressure_class); | |
7236 | } | |
7237 | } | |
7238 | ||
b8698a0f | 7239 | /* Initialize some global state for the scheduler. This function works |
e855c69d AB |
7240 | with the common data shared between all the schedulers. It is called |
7241 | from the scheduler specific initialization routine. */ | |
8c660648 | 7242 | |
b4ead7d4 | 7243 | void |
10d22567 | 7244 | sched_init (void) |
8c660648 | 7245 | { |
63de6c74 | 7246 | /* Disable speculative loads in their presence if cc0 defined. */ |
058eb3b0 | 7247 | if (HAVE_cc0) |
8c660648 | 7248 | flag_schedule_speculative_load = 0; |
8c660648 | 7249 | |
ac44248e DM |
7250 | if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON)) |
7251 | targetm.sched.dispatch_do (NULL, DISPATCH_INIT); | |
7942e47e | 7252 | |
f20f2613 VM |
7253 | if (live_range_shrinkage_p) |
7254 | sched_pressure = SCHED_PRESSURE_WEIGHTED; | |
7255 | else if (flag_sched_pressure | |
7256 | && !reload_completed | |
7257 | && common_sched_info->sched_pass_id == SCHED_RGN_PASS) | |
c881de02 RS |
7258 | sched_pressure = ((enum sched_pressure_algorithm) |
7259 | PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM)); | |
60867e8c RS |
7260 | else |
7261 | sched_pressure = SCHED_PRESSURE_NONE; | |
7942e47e | 7262 | |
7a1aca9c | 7263 | if (sched_pressure != SCHED_PRESSURE_NONE) |
8d49e7ef | 7264 | ira_setup_eliminable_regset (); |
ce18efcb | 7265 | |
496d7bb0 MK |
7266 | /* Initialize SPEC_INFO. */ |
7267 | if (targetm.sched.set_sched_flags) | |
7268 | { | |
7269 | spec_info = &spec_info_var; | |
7270 | targetm.sched.set_sched_flags (spec_info); | |
e855c69d AB |
7271 | |
7272 | if (spec_info->mask != 0) | |
7273 | { | |
7274 | spec_info->data_weakness_cutoff = | |
7275 | (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) * MAX_DEP_WEAK) / 100; | |
7276 | spec_info->control_weakness_cutoff = | |
7277 | (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) | |
7278 | * REG_BR_PROB_BASE) / 100; | |
7279 | } | |
496d7bb0 | 7280 | else |
6fc0bb99 | 7281 | /* So we won't read anything accidentally. */ |
e855c69d AB |
7282 | spec_info = NULL; |
7283 | ||
496d7bb0 MK |
7284 | } |
7285 | else | |
6fc0bb99 | 7286 | /* So we won't read anything accidentally. */ |
496d7bb0 MK |
7287 | spec_info = 0; |
7288 | ||
63de6c74 | 7289 | /* Initialize issue_rate. */ |
c237e94a | 7290 | if (targetm.sched.issue_rate) |
5fd9b178 | 7291 | issue_rate = targetm.sched.issue_rate (); |
c237e94a ZW |
7292 | else |
7293 | issue_rate = 1; | |
8c660648 | 7294 | |
d205caad MK |
7295 | if (targetm.sched.first_cycle_multipass_dfa_lookahead |
7296 | /* Don't use max_issue with reg_pressure scheduling. Multipass | |
7297 | scheduling and reg_pressure scheduling undo each other's decisions. */ | |
7298 | && sched_pressure == SCHED_PRESSURE_NONE) | |
e855c69d AB |
7299 | dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead (); |
7300 | else | |
7301 | dfa_lookahead = 0; | |
fae15c93 | 7302 | |
8aec463b MK |
7303 | /* Set to "0" so that we recalculate. */ |
7304 | max_lookahead_tries = 0; | |
7305 | ||
fa0aee89 PB |
7306 | if (targetm.sched.init_dfa_pre_cycle_insn) |
7307 | targetm.sched.init_dfa_pre_cycle_insn (); | |
1d088dee | 7308 | |
fa0aee89 PB |
7309 | if (targetm.sched.init_dfa_post_cycle_insn) |
7310 | targetm.sched.init_dfa_post_cycle_insn (); | |
1d088dee | 7311 | |
fa0aee89 PB |
7312 | dfa_start (); |
7313 | dfa_state_size = state_size (); | |
fae15c93 | 7314 | |
e855c69d | 7315 | init_alias_analysis (); |
f77e39fc | 7316 | |
26965010 BS |
7317 | if (!sched_no_dce) |
7318 | df_set_flags (DF_LR_RUN_DCE); | |
e855c69d | 7319 | df_note_add_problem (); |
f77e39fc | 7320 | |
e855c69d AB |
7321 | /* More problems needed for interloop dep calculation in SMS. */ |
7322 | if (common_sched_info->sched_pass_id == SCHED_SMS_PASS) | |
7323 | { | |
7324 | df_rd_add_problem (); | |
7325 | df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN); | |
7326 | } | |
7a403706 | 7327 | |
e855c69d | 7328 | df_analyze (); |
b8698a0f L |
7329 | |
7330 | /* Do not run DCE after reload, as this can kill nops inserted | |
e855c69d AB |
7331 | by bundling. */ |
7332 | if (reload_completed) | |
7333 | df_clear_flags (DF_LR_RUN_DCE); | |
a88f02e7 | 7334 | |
e855c69d | 7335 | regstat_compute_calls_crossed (); |
a88f02e7 | 7336 | |
38f8b050 JR |
7337 | if (targetm.sched.init_global) |
7338 | targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1); | |
a88f02e7 | 7339 | |
c4cd7435 | 7340 | alloc_global_sched_pressure_data (); |
b8698a0f | 7341 | |
e855c69d AB |
7342 | curr_state = xmalloc (dfa_state_size); |
7343 | } | |
58565a33 | 7344 | |
e855c69d | 7345 | static void haifa_init_only_bb (basic_block, basic_block); |
496d7bb0 | 7346 | |
e855c69d AB |
7347 | /* Initialize data structures specific to the Haifa scheduler. */ |
7348 | void | |
7349 | haifa_sched_init (void) | |
7350 | { | |
7351 | setup_sched_dump (); | |
7352 | sched_init (); | |
7353 | ||
9771b263 | 7354 | scheduled_insns.create (0); |
9b69cf83 | 7355 | |
e855c69d AB |
7356 | if (spec_info != NULL) |
7357 | { | |
7358 | sched_deps_info->use_deps_list = 1; | |
7359 | sched_deps_info->generate_spec_deps = 1; | |
7360 | } | |
7361 | ||
7362 | /* Initialize luids, dependency caches, target and h_i_d for the | |
7363 | whole function. */ | |
7364 | { | |
e855c69d AB |
7365 | sched_init_bbs (); |
7366 | ||
8c681247 TS |
7367 | auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun)); |
7368 | basic_block bb; | |
11cd3bed | 7369 | FOR_EACH_BB_FN (bb, cfun) |
9771b263 | 7370 | bbs.quick_push (bb); |
a95b23b4 | 7371 | sched_init_luids (bbs); |
e855c69d AB |
7372 | sched_deps_init (true); |
7373 | sched_extend_target (); | |
a95b23b4 | 7374 | haifa_init_h_i_d (bbs); |
e855c69d AB |
7375 | } |
7376 | ||
7377 | sched_init_only_bb = haifa_init_only_bb; | |
7378 | sched_split_block = sched_split_block_1; | |
7379 | sched_create_empty_bb = sched_create_empty_bb_1; | |
e2f6ff94 MK |
7380 | haifa_recovery_bb_ever_added_p = false; |
7381 | ||
e855c69d AB |
7382 | nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0; |
7383 | before_recovery = 0; | |
7384 | after_recovery = 0; | |
06d7e8e7 BS |
7385 | |
7386 | modulo_ii = 0; | |
e855c69d | 7387 | } |
8c660648 | 7388 | |
e855c69d | 7389 | /* Finish work with the data specific to the Haifa scheduler. */ |
a88f02e7 | 7390 | void |
e855c69d | 7391 | haifa_sched_finish (void) |
a88f02e7 | 7392 | { |
e855c69d AB |
7393 | sched_create_empty_bb = NULL; |
7394 | sched_split_block = NULL; | |
7395 | sched_init_only_bb = NULL; | |
58565a33 | 7396 | |
496d7bb0 MK |
7397 | if (spec_info && spec_info->dump) |
7398 | { | |
7399 | char c = reload_completed ? 'a' : 'b'; | |
7400 | ||
7401 | fprintf (spec_info->dump, | |
7402 | ";; %s:\n", current_function_name ()); | |
7403 | ||
7404 | fprintf (spec_info->dump, | |
7405 | ";; Procedure %cr-begin-data-spec motions == %d\n", | |
7406 | c, nr_begin_data); | |
7407 | fprintf (spec_info->dump, | |
7408 | ";; Procedure %cr-be-in-data-spec motions == %d\n", | |
7409 | c, nr_be_in_data); | |
7410 | fprintf (spec_info->dump, | |
7411 | ";; Procedure %cr-begin-control-spec motions == %d\n", | |
7412 | c, nr_begin_control); | |
7413 | fprintf (spec_info->dump, | |
7414 | ";; Procedure %cr-be-in-control-spec motions == %d\n", | |
7415 | c, nr_be_in_control); | |
7416 | } | |
7417 | ||
9771b263 | 7418 | scheduled_insns.release (); |
9b69cf83 | 7419 | |
e855c69d AB |
7420 | /* Finalize h_i_d, dependency caches, and luids for the whole |
7421 | function. Target will be finalized in md_global_finish (). */ | |
7422 | sched_deps_finish (); | |
7423 | sched_finish_luids (); | |
7424 | current_sched_info = NULL; | |
df554b0e | 7425 | insn_queue = NULL; |
e855c69d AB |
7426 | sched_finish (); |
7427 | } | |
7428 | ||
b8698a0f | 7429 | /* Free global data used during insn scheduling. This function works with |
e855c69d AB |
7430 | the common data shared between the schedulers. */ |
7431 | ||
7432 | void | |
7433 | sched_finish (void) | |
7434 | { | |
7435 | haifa_finish_h_i_d (); | |
c4cd7435 | 7436 | free_global_sched_pressure_data (); |
e855c69d AB |
7437 | free (curr_state); |
7438 | ||
38f8b050 JR |
7439 | if (targetm.sched.finish_global) |
7440 | targetm.sched.finish_global (sched_dump, sched_verbose); | |
e855c69d AB |
7441 | |
7442 | end_alias_analysis (); | |
7443 | ||
7444 | regstat_free_calls_crossed (); | |
7445 | ||
7446 | dfa_finish (); | |
8c660648 | 7447 | } |
63f54b1a | 7448 | |
26965010 BS |
7449 | /* Free all delay_pair structures that were recorded. */ |
7450 | void | |
7451 | free_delay_pairs (void) | |
7452 | { | |
c203e8a7 | 7453 | if (delay_htab) |
26965010 | 7454 | { |
c203e8a7 TS |
7455 | delay_htab->empty (); |
7456 | delay_htab_i2->empty (); | |
26965010 BS |
7457 | } |
7458 | } | |
7459 | ||
63f54b1a | 7460 | /* Fix INSN_TICKs of the instructions in the current block as well as |
917f1b7e | 7461 | INSN_TICKs of their dependents. |
63f54b1a MK |
7462 | HEAD and TAIL are the begin and the end of the current scheduled block. */ |
7463 | static void | |
f57aa6b0 | 7464 | fix_inter_tick (rtx_insn *head, rtx_insn *tail) |
63f54b1a MK |
7465 | { |
7466 | /* Set of instructions with corrected INSN_TICK. */ | |
8f9b31f7 | 7467 | auto_bitmap processed; |
e2f6ff94 MK |
7468 | /* ??? It is doubtful if we should assume that cycle advance happens on |
7469 | basic block boundaries. Basically insns that are unconditionally ready | |
7470 | on the start of the block are more preferable then those which have | |
7471 | a one cycle dependency over insn from the previous block. */ | |
63f54b1a MK |
7472 | int next_clock = clock_var + 1; |
7473 | ||
63f54b1a MK |
7474 | /* Iterates over scheduled instructions and fix their INSN_TICKs and |
7475 | INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent | |
7476 | across different blocks. */ | |
7477 | for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head)) | |
7478 | { | |
7479 | if (INSN_P (head)) | |
7480 | { | |
7481 | int tick; | |
e2f6ff94 MK |
7482 | sd_iterator_def sd_it; |
7483 | dep_t dep; | |
b8698a0f | 7484 | |
63f54b1a MK |
7485 | tick = INSN_TICK (head); |
7486 | gcc_assert (tick >= MIN_TICK); | |
b8698a0f | 7487 | |
63f54b1a | 7488 | /* Fix INSN_TICK of instruction from just scheduled block. */ |
8f9b31f7 | 7489 | if (bitmap_set_bit (processed, INSN_LUID (head))) |
63f54b1a | 7490 | { |
63f54b1a | 7491 | tick -= next_clock; |
b8698a0f | 7492 | |
63f54b1a MK |
7493 | if (tick < MIN_TICK) |
7494 | tick = MIN_TICK; | |
b8698a0f L |
7495 | |
7496 | INSN_TICK (head) = tick; | |
63f54b1a | 7497 | } |
b8698a0f | 7498 | |
35d59da7 UB |
7499 | if (DEBUG_INSN_P (head)) |
7500 | continue; | |
7501 | ||
e2f6ff94 | 7502 | FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep) |
63f54b1a | 7503 | { |
9b517712 | 7504 | rtx_insn *next; |
b8698a0f | 7505 | |
e2f6ff94 | 7506 | next = DEP_CON (dep); |
63f54b1a MK |
7507 | tick = INSN_TICK (next); |
7508 | ||
7509 | if (tick != INVALID_TICK | |
7510 | /* If NEXT has its INSN_TICK calculated, fix it. | |
7511 | If not - it will be properly calculated from | |
7512 | scratch later in fix_tick_ready. */ | |
8f9b31f7 | 7513 | && bitmap_set_bit (processed, INSN_LUID (next))) |
63f54b1a | 7514 | { |
63f54b1a | 7515 | tick -= next_clock; |
b8698a0f | 7516 | |
63f54b1a MK |
7517 | if (tick < MIN_TICK) |
7518 | tick = MIN_TICK; | |
b8698a0f | 7519 | |
63f54b1a MK |
7520 | if (tick > INTER_TICK (next)) |
7521 | INTER_TICK (next) = tick; | |
7522 | else | |
7523 | tick = INTER_TICK (next); | |
e2f6ff94 | 7524 | |
63f54b1a MK |
7525 | INSN_TICK (next) = tick; |
7526 | } | |
7527 | } | |
7528 | } | |
7529 | } | |
63f54b1a | 7530 | } |
e855c69d | 7531 | |
63f54b1a MK |
7532 | /* Check if NEXT is ready to be added to the ready or queue list. |
7533 | If "yes", add it to the proper list. | |
7534 | Returns: | |
7535 | -1 - is not ready yet, | |
7536 | 0 - added to the ready list, | |
7537 | 0 < N - queued for N cycles. */ | |
7538 | int | |
ce1ce33a | 7539 | try_ready (rtx_insn *next) |
b8698a0f | 7540 | { |
b953c2b8 | 7541 | ds_t old_ts, new_ts; |
63f54b1a | 7542 | |
b953c2b8 | 7543 | old_ts = TODO_SPEC (next); |
63f54b1a | 7544 | |
1a83e602 BS |
7545 | gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED)) |
7546 | && (old_ts == HARD_DEP | |
7547 | || old_ts == DEP_POSTPONED | |
e2724e63 | 7548 | || (old_ts & SPECULATIVE) |
1a83e602 | 7549 | || old_ts == DEP_CONTROL)); |
e2f6ff94 | 7550 | |
1a83e602 | 7551 | new_ts = recompute_todo_spec (next, false); |
b198261f | 7552 | |
1a83e602 | 7553 | if (new_ts & (HARD_DEP | DEP_POSTPONED)) |
e2724e63 | 7554 | gcc_assert (new_ts == old_ts |
496d7bb0 MK |
7555 | && QUEUE_INDEX (next) == QUEUE_NOWHERE); |
7556 | else if (current_sched_info->new_ready) | |
b953c2b8 | 7557 | new_ts = current_sched_info->new_ready (next, new_ts); |
496d7bb0 | 7558 | |
b198261f | 7559 | /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might |
496d7bb0 MK |
7560 | have its original pattern or changed (speculative) one. This is due |
7561 | to changing ebb in region scheduling. | |
7562 | * But if (old_ts & SPECULATIVE), then we are pretty sure that insn | |
7563 | has speculative pattern. | |
b198261f | 7564 | |
b953c2b8 | 7565 | We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because |
496d7bb0 MK |
7566 | control-speculative NEXT could have been discarded by sched-rgn.c |
7567 | (the same case as when discarded by can_schedule_ready_p ()). */ | |
b198261f | 7568 | |
b953c2b8 BS |
7569 | if ((new_ts & SPECULATIVE) |
7570 | /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't | |
496d7bb0 | 7571 | need to change anything. */ |
b953c2b8 | 7572 | && new_ts != old_ts) |
496d7bb0 MK |
7573 | { |
7574 | int res; | |
7575 | rtx new_pat; | |
b8698a0f | 7576 | |
e2724e63 | 7577 | gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE)); |
b8698a0f | 7578 | |
b953c2b8 | 7579 | res = haifa_speculate_insn (next, new_ts, &new_pat); |
b8698a0f | 7580 | |
496d7bb0 MK |
7581 | switch (res) |
7582 | { | |
7583 | case -1: | |
7584 | /* It would be nice to change DEP_STATUS of all dependences, | |
b953c2b8 | 7585 | which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP, |
496d7bb0 | 7586 | so we won't reanalyze anything. */ |
b953c2b8 | 7587 | new_ts = HARD_DEP; |
496d7bb0 | 7588 | break; |
b8698a0f | 7589 | |
496d7bb0 MK |
7590 | case 0: |
7591 | /* We follow the rule, that every speculative insn | |
7592 | has non-null ORIG_PAT. */ | |
7593 | if (!ORIG_PAT (next)) | |
7594 | ORIG_PAT (next) = PATTERN (next); | |
7595 | break; | |
b8698a0f L |
7596 | |
7597 | case 1: | |
496d7bb0 MK |
7598 | if (!ORIG_PAT (next)) |
7599 | /* If we gonna to overwrite the original pattern of insn, | |
7600 | save it. */ | |
7601 | ORIG_PAT (next) = PATTERN (next); | |
b8698a0f | 7602 | |
e2724e63 BS |
7603 | res = haifa_change_pattern (next, new_pat); |
7604 | gcc_assert (res); | |
496d7bb0 | 7605 | break; |
b8698a0f | 7606 | |
496d7bb0 MK |
7607 | default: |
7608 | gcc_unreachable (); | |
7609 | } | |
7610 | } | |
b8698a0f | 7611 | |
b953c2b8 BS |
7612 | /* We need to restore pattern only if (new_ts == 0), because otherwise it is |
7613 | either correct (new_ts & SPECULATIVE), | |
7614 | or we simply don't care (new_ts & HARD_DEP). */ | |
b8698a0f | 7615 | |
496d7bb0 | 7616 | gcc_assert (!ORIG_PAT (next) |
d7bfd907 | 7617 | || !IS_SPECULATION_BRANCHY_CHECK_P (next)); |
b8698a0f | 7618 | |
b953c2b8 BS |
7619 | TODO_SPEC (next) = new_ts; |
7620 | ||
1a83e602 | 7621 | if (new_ts & (HARD_DEP | DEP_POSTPONED)) |
496d7bb0 MK |
7622 | { |
7623 | /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because | |
7624 | control-speculative NEXT could have been discarded by sched-rgn.c | |
7625 | (the same case as when discarded by can_schedule_ready_p ()). */ | |
7626 | /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/ | |
b8698a0f | 7627 | |
496d7bb0 | 7628 | change_queue_index (next, QUEUE_NOWHERE); |
e2724e63 | 7629 | |
496d7bb0 MK |
7630 | return -1; |
7631 | } | |
b953c2b8 | 7632 | else if (!(new_ts & BEGIN_SPEC) |
e2724e63 BS |
7633 | && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX |
7634 | && !IS_SPECULATION_CHECK_P (next)) | |
b8698a0f | 7635 | /* We should change pattern of every previously speculative |
682b6a9e | 7636 | instruction - and we determine if NEXT was speculative by using |
d7bfd907 MK |
7637 | ORIG_PAT field. Except one case - speculation checks have ORIG_PAT |
7638 | pat too, so skip them. */ | |
682b6a9e | 7639 | { |
e2724e63 BS |
7640 | bool success = haifa_change_pattern (next, ORIG_PAT (next)); |
7641 | gcc_assert (success); | |
682b6a9e MK |
7642 | ORIG_PAT (next) = 0; |
7643 | } | |
496d7bb0 MK |
7644 | |
7645 | if (sched_verbose >= 2) | |
b8698a0f | 7646 | { |
496d7bb0 MK |
7647 | fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s", |
7648 | (*current_sched_info->print_insn) (next, 0)); | |
b8698a0f | 7649 | |
496d7bb0 MK |
7650 | if (spec_info && spec_info->dump) |
7651 | { | |
b953c2b8 | 7652 | if (new_ts & BEGIN_DATA) |
496d7bb0 | 7653 | fprintf (spec_info->dump, "; data-spec;"); |
b953c2b8 | 7654 | if (new_ts & BEGIN_CONTROL) |
496d7bb0 | 7655 | fprintf (spec_info->dump, "; control-spec;"); |
b953c2b8 | 7656 | if (new_ts & BE_IN_CONTROL) |
496d7bb0 MK |
7657 | fprintf (spec_info->dump, "; in-control-spec;"); |
7658 | } | |
e2724e63 BS |
7659 | if (TODO_SPEC (next) & DEP_CONTROL) |
7660 | fprintf (sched_dump, " predicated"); | |
496d7bb0 | 7661 | fprintf (sched_dump, "\n"); |
b8698a0f L |
7662 | } |
7663 | ||
496d7bb0 | 7664 | adjust_priority (next); |
b8698a0f | 7665 | |
496d7bb0 MK |
7666 | return fix_tick_ready (next); |
7667 | } | |
7668 | ||
7669 | /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */ | |
7670 | static int | |
ce1ce33a | 7671 | fix_tick_ready (rtx_insn *next) |
63f54b1a | 7672 | { |
63f54b1a MK |
7673 | int tick, delay; |
7674 | ||
b2cddfc8 | 7675 | if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK)) |
63f54b1a MK |
7676 | { |
7677 | int full_p; | |
e2f6ff94 MK |
7678 | sd_iterator_def sd_it; |
7679 | dep_t dep; | |
63f54b1a MK |
7680 | |
7681 | tick = INSN_TICK (next); | |
496d7bb0 | 7682 | /* if tick is not equal to INVALID_TICK, then update |
63f54b1a | 7683 | INSN_TICK of NEXT with the most recent resolved dependence |
917f1b7e | 7684 | cost. Otherwise, recalculate from scratch. */ |
b198261f MK |
7685 | full_p = (tick == INVALID_TICK); |
7686 | ||
e2f6ff94 | 7687 | FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep) |
b8698a0f | 7688 | { |
9b517712 | 7689 | rtx_insn *pro = DEP_PRO (dep); |
63f54b1a | 7690 | int tick1; |
b8698a0f | 7691 | |
63f54b1a | 7692 | gcc_assert (INSN_TICK (pro) >= MIN_TICK); |
496d7bb0 | 7693 | |
b198261f | 7694 | tick1 = INSN_TICK (pro) + dep_cost (dep); |
63f54b1a MK |
7695 | if (tick1 > tick) |
7696 | tick = tick1; | |
b198261f MK |
7697 | |
7698 | if (!full_p) | |
7699 | break; | |
63f54b1a | 7700 | } |
63f54b1a MK |
7701 | } |
7702 | else | |
7703 | tick = -1; | |
7704 | ||
7705 | INSN_TICK (next) = tick; | |
7706 | ||
7707 | delay = tick - clock_var; | |
b16abbcb | 7708 | if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion) |
63f54b1a MK |
7709 | delay = QUEUE_READY; |
7710 | ||
7711 | change_queue_index (next, delay); | |
496d7bb0 | 7712 | |
63f54b1a MK |
7713 | return delay; |
7714 | } | |
7715 | ||
7716 | /* Move NEXT to the proper queue list with (DELAY >= 1), | |
7717 | or add it to the ready list (DELAY == QUEUE_READY), | |
7718 | or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */ | |
7719 | static void | |
ce1ce33a | 7720 | change_queue_index (rtx_insn *next, int delay) |
63f54b1a MK |
7721 | { |
7722 | int i = QUEUE_INDEX (next); | |
7723 | ||
b8698a0f | 7724 | gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index |
63f54b1a MK |
7725 | && delay != 0); |
7726 | gcc_assert (i != QUEUE_SCHEDULED); | |
b8698a0f | 7727 | |
63f54b1a MK |
7728 | if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i) |
7729 | || (delay < 0 && delay == i)) | |
7730 | /* We have nothing to do. */ | |
7731 | return; | |
7732 | ||
917f1b7e | 7733 | /* Remove NEXT from wherever it is now. */ |
63f54b1a MK |
7734 | if (i == QUEUE_READY) |
7735 | ready_remove_insn (next); | |
7736 | else if (i >= 0) | |
7737 | queue_remove (next); | |
b8698a0f | 7738 | |
63f54b1a MK |
7739 | /* Add it to the proper place. */ |
7740 | if (delay == QUEUE_READY) | |
7741 | ready_add (readyp, next, false); | |
7742 | else if (delay >= 1) | |
ec4efea9 | 7743 | queue_insn (next, delay, "change queue index"); |
b8698a0f | 7744 | |
63f54b1a | 7745 | if (sched_verbose >= 2) |
b8698a0f | 7746 | { |
63f54b1a MK |
7747 | fprintf (sched_dump, ";;\t\ttick updated: insn %s", |
7748 | (*current_sched_info->print_insn) (next, 0)); | |
b8698a0f | 7749 | |
63f54b1a MK |
7750 | if (delay == QUEUE_READY) |
7751 | fprintf (sched_dump, " into ready\n"); | |
7752 | else if (delay >= 1) | |
7753 | fprintf (sched_dump, " into queue with cost=%d\n", delay); | |
7754 | else | |
7755 | fprintf (sched_dump, " removed from ready or queue lists\n"); | |
7756 | } | |
7757 | } | |
7758 | ||
e855c69d | 7759 | static int sched_ready_n_insns = -1; |
496d7bb0 | 7760 | |
e855c69d AB |
7761 | /* Initialize per region data structures. */ |
7762 | void | |
7763 | sched_extend_ready_list (int new_sched_ready_n_insns) | |
496d7bb0 MK |
7764 | { |
7765 | int i; | |
7766 | ||
e855c69d AB |
7767 | if (sched_ready_n_insns == -1) |
7768 | /* At the first call we need to initialize one more choice_stack | |
7769 | entry. */ | |
7770 | { | |
7771 | i = 0; | |
7772 | sched_ready_n_insns = 0; | |
9771b263 | 7773 | scheduled_insns.reserve (new_sched_ready_n_insns); |
e855c69d AB |
7774 | } |
7775 | else | |
7776 | i = sched_ready_n_insns + 1; | |
496d7bb0 | 7777 | |
e855c69d | 7778 | ready.veclen = new_sched_ready_n_insns + issue_rate; |
ce1ce33a | 7779 | ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen); |
496d7bb0 | 7780 | |
e855c69d | 7781 | gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns); |
496d7bb0 | 7782 | |
4960a0cb MK |
7783 | ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns, |
7784 | sched_ready_n_insns, | |
7785 | sizeof (*ready_try)); | |
496d7bb0 | 7786 | |
e855c69d AB |
7787 | /* We allocate +1 element to save initial state in the choice_stack[0] |
7788 | entry. */ | |
7789 | choice_stack = XRESIZEVEC (struct choice_entry, choice_stack, | |
7790 | new_sched_ready_n_insns + 1); | |
e2f6ff94 | 7791 | |
e855c69d | 7792 | for (; i <= new_sched_ready_n_insns; i++) |
894fd6f2 MK |
7793 | { |
7794 | choice_stack[i].state = xmalloc (dfa_state_size); | |
7795 | ||
7796 | if (targetm.sched.first_cycle_multipass_init) | |
7797 | targetm.sched.first_cycle_multipass_init (&(choice_stack[i] | |
7798 | .target_data)); | |
7799 | } | |
496d7bb0 | 7800 | |
e855c69d | 7801 | sched_ready_n_insns = new_sched_ready_n_insns; |
496d7bb0 MK |
7802 | } |
7803 | ||
e855c69d AB |
7804 | /* Free per region data structures. */ |
7805 | void | |
7806 | sched_finish_ready_list (void) | |
a0d33ff8 | 7807 | { |
e855c69d | 7808 | int i; |
a0d33ff8 | 7809 | |
e855c69d AB |
7810 | free (ready.vec); |
7811 | ready.vec = NULL; | |
7812 | ready.veclen = 0; | |
a0d33ff8 | 7813 | |
e855c69d AB |
7814 | free (ready_try); |
7815 | ready_try = NULL; | |
496d7bb0 | 7816 | |
e855c69d | 7817 | for (i = 0; i <= sched_ready_n_insns; i++) |
894fd6f2 MK |
7818 | { |
7819 | if (targetm.sched.first_cycle_multipass_fini) | |
7820 | targetm.sched.first_cycle_multipass_fini (&(choice_stack[i] | |
7821 | .target_data)); | |
7822 | ||
7823 | free (choice_stack [i].state); | |
7824 | } | |
e855c69d AB |
7825 | free (choice_stack); |
7826 | choice_stack = NULL; | |
a0d33ff8 | 7827 | |
e855c69d | 7828 | sched_ready_n_insns = -1; |
496d7bb0 MK |
7829 | } |
7830 | ||
e855c69d AB |
7831 | static int |
7832 | haifa_luid_for_non_insn (rtx x) | |
496d7bb0 | 7833 | { |
e855c69d AB |
7834 | gcc_assert (NOTE_P (x) || LABEL_P (x)); |
7835 | ||
7836 | return 0; | |
496d7bb0 MK |
7837 | } |
7838 | ||
7839 | /* Generates recovery code for INSN. */ | |
7840 | static void | |
ce1ce33a | 7841 | generate_recovery_code (rtx_insn *insn) |
496d7bb0 MK |
7842 | { |
7843 | if (TODO_SPEC (insn) & BEGIN_SPEC) | |
7844 | begin_speculative_block (insn); | |
b8698a0f | 7845 | |
496d7bb0 MK |
7846 | /* Here we have insn with no dependencies to |
7847 | instructions other then CHECK_SPEC ones. */ | |
b8698a0f | 7848 | |
496d7bb0 MK |
7849 | if (TODO_SPEC (insn) & BE_IN_SPEC) |
7850 | add_to_speculative_block (insn); | |
7851 | } | |
7852 | ||
7853 | /* Helper function. | |
7854 | Tries to add speculative dependencies of type FS between instructions | |
b198261f | 7855 | in deps_list L and TWIN. */ |
496d7bb0 | 7856 | static void |
90831096 | 7857 | process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs) |
496d7bb0 | 7858 | { |
e2f6ff94 MK |
7859 | sd_iterator_def sd_it; |
7860 | dep_t dep; | |
b198261f | 7861 | |
e2f6ff94 | 7862 | FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep) |
496d7bb0 MK |
7863 | { |
7864 | ds_t ds; | |
0e0ce50d | 7865 | rtx_insn *consumer; |
496d7bb0 | 7866 | |
e2f6ff94 | 7867 | consumer = DEP_CON (dep); |
496d7bb0 | 7868 | |
e2f6ff94 | 7869 | ds = DEP_STATUS (dep); |
496d7bb0 | 7870 | |
682b6a9e MK |
7871 | if (/* If we want to create speculative dep. */ |
7872 | fs | |
7873 | /* And we can do that because this is a true dep. */ | |
7874 | && (ds & DEP_TYPES) == DEP_TRUE) | |
7875 | { | |
7876 | gcc_assert (!(ds & BE_IN_SPEC)); | |
7877 | ||
917f1b7e | 7878 | if (/* If this dep can be overcome with 'begin speculation'. */ |
682b6a9e MK |
7879 | ds & BEGIN_SPEC) |
7880 | /* Then we have a choice: keep the dep 'begin speculative' | |
7881 | or transform it into 'be in speculative'. */ | |
7882 | { | |
7883 | if (/* In try_ready we assert that if insn once became ready | |
7884 | it can be removed from the ready (or queue) list only | |
7885 | due to backend decision. Hence we can't let the | |
7886 | probability of the speculative dep to decrease. */ | |
e855c69d | 7887 | ds_weak (ds) <= ds_weak (fs)) |
93b4b4cc MK |
7888 | { |
7889 | ds_t new_ds; | |
7890 | ||
7891 | new_ds = (ds & ~BEGIN_SPEC) | fs; | |
b8698a0f | 7892 | |
93b4b4cc MK |
7893 | if (/* consumer can 'be in speculative'. */ |
7894 | sched_insn_is_legitimate_for_speculation_p (consumer, | |
7895 | new_ds)) | |
7896 | /* Transform it to be in speculative. */ | |
7897 | ds = new_ds; | |
7898 | } | |
682b6a9e MK |
7899 | } |
7900 | else | |
7901 | /* Mark the dep as 'be in speculative'. */ | |
7902 | ds |= fs; | |
7903 | } | |
496d7bb0 | 7904 | |
e2f6ff94 MK |
7905 | { |
7906 | dep_def _new_dep, *new_dep = &_new_dep; | |
7907 | ||
7908 | init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds); | |
7909 | sd_add_dep (new_dep, false); | |
7910 | } | |
496d7bb0 MK |
7911 | } |
7912 | } | |
7913 | ||
7914 | /* Generates recovery code for BEGIN speculative INSN. */ | |
7915 | static void | |
ce1ce33a | 7916 | begin_speculative_block (rtx_insn *insn) |
496d7bb0 MK |
7917 | { |
7918 | if (TODO_SPEC (insn) & BEGIN_DATA) | |
b8698a0f | 7919 | nr_begin_data++; |
496d7bb0 MK |
7920 | if (TODO_SPEC (insn) & BEGIN_CONTROL) |
7921 | nr_begin_control++; | |
7922 | ||
7923 | create_check_block_twin (insn, false); | |
7924 | ||
7925 | TODO_SPEC (insn) &= ~BEGIN_SPEC; | |
7926 | } | |
7927 | ||
f57aa6b0 | 7928 | static void haifa_init_insn (rtx_insn *); |
e855c69d | 7929 | |
496d7bb0 MK |
7930 | /* Generates recovery code for BE_IN speculative INSN. */ |
7931 | static void | |
6144a836 | 7932 | add_to_speculative_block (rtx_insn *insn) |
496d7bb0 MK |
7933 | { |
7934 | ds_t ts; | |
e2f6ff94 MK |
7935 | sd_iterator_def sd_it; |
7936 | dep_t dep; | |
30a7199a | 7937 | auto_vec<rtx_insn *, 10> twins; |
496d7bb0 MK |
7938 | |
7939 | ts = TODO_SPEC (insn); | |
7940 | gcc_assert (!(ts & ~BE_IN_SPEC)); | |
7941 | ||
7942 | if (ts & BE_IN_DATA) | |
7943 | nr_be_in_data++; | |
7944 | if (ts & BE_IN_CONTROL) | |
7945 | nr_be_in_control++; | |
7946 | ||
7947 | TODO_SPEC (insn) &= ~BE_IN_SPEC; | |
7948 | gcc_assert (!TODO_SPEC (insn)); | |
b8698a0f | 7949 | |
496d7bb0 MK |
7950 | DONE_SPEC (insn) |= ts; |
7951 | ||
7952 | /* First we convert all simple checks to branchy. */ | |
e2f6ff94 MK |
7953 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
7954 | sd_iterator_cond (&sd_it, &dep);) | |
496d7bb0 | 7955 | { |
9b517712 | 7956 | rtx_insn *check = DEP_PRO (dep); |
496d7bb0 | 7957 | |
d7bfd907 | 7958 | if (IS_SPECULATION_SIMPLE_CHECK_P (check)) |
496d7bb0 MK |
7959 | { |
7960 | create_check_block_twin (check, true); | |
b198261f MK |
7961 | |
7962 | /* Restart search. */ | |
e2f6ff94 | 7963 | sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
496d7bb0 MK |
7964 | } |
7965 | else | |
b198261f | 7966 | /* Continue search. */ |
e2f6ff94 | 7967 | sd_iterator_next (&sd_it); |
496d7bb0 MK |
7968 | } |
7969 | ||
8c681247 | 7970 | auto_vec<rtx_insn *> priorities_roots; |
916fa4f0 | 7971 | clear_priorities (insn, &priorities_roots); |
e2f6ff94 MK |
7972 | |
7973 | while (1) | |
496d7bb0 | 7974 | { |
9b517712 | 7975 | rtx_insn *check, *twin; |
496d7bb0 MK |
7976 | basic_block rec; |
7977 | ||
e2f6ff94 MK |
7978 | /* Get the first backward dependency of INSN. */ |
7979 | sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
7980 | if (!sd_iterator_cond (&sd_it, &dep)) | |
7981 | /* INSN has no backward dependencies left. */ | |
7982 | break; | |
496d7bb0 | 7983 | |
e2f6ff94 MK |
7984 | gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == 0 |
7985 | && (DEP_STATUS (dep) & BE_IN_SPEC) != 0 | |
7986 | && (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE); | |
b198261f | 7987 | |
e2f6ff94 | 7988 | check = DEP_PRO (dep); |
d7bfd907 MK |
7989 | |
7990 | gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check) | |
496d7bb0 | 7991 | && QUEUE_INDEX (check) == QUEUE_NOWHERE); |
e2f6ff94 | 7992 | |
496d7bb0 | 7993 | rec = BLOCK_FOR_INSN (check); |
e2f6ff94 | 7994 | |
05742530 | 7995 | twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec)); |
e855c69d | 7996 | haifa_init_insn (twin); |
496d7bb0 | 7997 | |
e2f6ff94 | 7998 | sd_copy_back_deps (twin, insn, true); |
496d7bb0 MK |
7999 | |
8000 | if (sched_verbose && spec_info->dump) | |
8001 | /* INSN_BB (insn) isn't determined for twin insns yet. | |
8002 | So we can't use current_sched_info->print_insn. */ | |
8003 | fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n", | |
8004 | INSN_UID (twin), rec->index); | |
8005 | ||
30a7199a | 8006 | twins.safe_push (twin); |
496d7bb0 | 8007 | |
917f1b7e | 8008 | /* Add dependences between TWIN and all appropriate |
496d7bb0 | 8009 | instructions from REC. */ |
e2f6ff94 MK |
8010 | FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep) |
8011 | { | |
9b517712 | 8012 | rtx_insn *pro = DEP_PRO (dep); |
b198261f | 8013 | |
e2f6ff94 MK |
8014 | gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE); |
8015 | ||
8016 | /* INSN might have dependencies from the instructions from | |
8017 | several recovery blocks. At this iteration we process those | |
8018 | producers that reside in REC. */ | |
8019 | if (BLOCK_FOR_INSN (pro) == rec) | |
8020 | { | |
8021 | dep_def _new_dep, *new_dep = &_new_dep; | |
8022 | ||
8023 | init_dep (new_dep, pro, twin, REG_DEP_TRUE); | |
8024 | sd_add_dep (new_dep, false); | |
496d7bb0 | 8025 | } |
496d7bb0 | 8026 | } |
496d7bb0 | 8027 | |
e2f6ff94 | 8028 | process_insn_forw_deps_be_in_spec (insn, twin, ts); |
496d7bb0 | 8029 | |
b198261f | 8030 | /* Remove all dependencies between INSN and insns in REC. */ |
e2f6ff94 MK |
8031 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
8032 | sd_iterator_cond (&sd_it, &dep);) | |
496d7bb0 | 8033 | { |
9b517712 | 8034 | rtx_insn *pro = DEP_PRO (dep); |
b198261f | 8035 | |
e2f6ff94 MK |
8036 | if (BLOCK_FOR_INSN (pro) == rec) |
8037 | sd_delete_dep (sd_it); | |
496d7bb0 | 8038 | else |
e2f6ff94 | 8039 | sd_iterator_next (&sd_it); |
496d7bb0 MK |
8040 | } |
8041 | } | |
496d7bb0 | 8042 | |
b198261f MK |
8043 | /* We couldn't have added the dependencies between INSN and TWINS earlier |
8044 | because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */ | |
30a7199a TS |
8045 | unsigned int i; |
8046 | rtx_insn *twin; | |
8047 | FOR_EACH_VEC_ELT_REVERSE (twins, i, twin) | |
496d7bb0 | 8048 | { |
30a7199a | 8049 | dep_def _new_dep, *new_dep = &_new_dep; |
496d7bb0 | 8050 | |
30a7199a TS |
8051 | init_dep (new_dep, insn, twin, REG_DEP_OUTPUT); |
8052 | sd_add_dep (new_dep, false); | |
496d7bb0 | 8053 | } |
916fa4f0 MK |
8054 | |
8055 | calc_priorities (priorities_roots); | |
496d7bb0 MK |
8056 | } |
8057 | ||
8058 | /* Extends and fills with zeros (only the new part) array pointed to by P. */ | |
8059 | void * | |
8060 | xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size) | |
8061 | { | |
8062 | gcc_assert (new_nmemb >= old_nmemb); | |
8063 | p = XRESIZEVAR (void, p, new_nmemb * size); | |
8064 | memset (((char *) p) + old_nmemb * size, 0, (new_nmemb - old_nmemb) * size); | |
8065 | return p; | |
8066 | } | |
8067 | ||
496d7bb0 MK |
8068 | /* Helper function. |
8069 | Find fallthru edge from PRED. */ | |
e855c69d | 8070 | edge |
0fd4b31d | 8071 | find_fallthru_edge_from (basic_block pred) |
496d7bb0 MK |
8072 | { |
8073 | edge e; | |
496d7bb0 MK |
8074 | basic_block succ; |
8075 | ||
8076 | succ = pred->next_bb; | |
8077 | gcc_assert (succ->prev_bb == pred); | |
8078 | ||
8079 | if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds)) | |
8080 | { | |
0fd4b31d NF |
8081 | e = find_fallthru_edge (pred->succs); |
8082 | ||
8083 | if (e) | |
8084 | { | |
50550600 | 8085 | gcc_assert (e->dest == succ || e->dest->index == EXIT_BLOCK); |
0fd4b31d NF |
8086 | return e; |
8087 | } | |
496d7bb0 MK |
8088 | } |
8089 | else | |
8090 | { | |
0fd4b31d NF |
8091 | e = find_fallthru_edge (succ->preds); |
8092 | ||
8093 | if (e) | |
8094 | { | |
8095 | gcc_assert (e->src == pred); | |
8096 | return e; | |
8097 | } | |
496d7bb0 MK |
8098 | } |
8099 | ||
8100 | return NULL; | |
8101 | } | |
8102 | ||
e855c69d AB |
8103 | /* Extend per basic block data structures. */ |
8104 | static void | |
8105 | sched_extend_bb (void) | |
8106 | { | |
e855c69d | 8107 | /* The following is done to keep current_sched_info->next_tail non null. */ |
9b517712 DM |
8108 | rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb); |
8109 | rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end; | |
cb5cb194 | 8110 | if (NEXT_INSN (end) == 0 |
e855c69d AB |
8111 | || (!NOTE_P (insn) |
8112 | && !LABEL_P (insn) | |
8113 | /* Don't emit a NOTE if it would end up before a BARRIER. */ | |
65f4b875 | 8114 | && !BARRIER_P (next_nondebug_insn (end)))) |
e855c69d | 8115 | { |
66e8df53 | 8116 | rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end); |
cb5cb194 | 8117 | /* Make note appear outside BB. */ |
e855c69d | 8118 | set_block_for_insn (note, NULL); |
1130d5e3 | 8119 | BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end; |
e855c69d AB |
8120 | } |
8121 | } | |
8122 | ||
8123 | /* Init per basic block data structures. */ | |
8124 | void | |
8125 | sched_init_bbs (void) | |
8126 | { | |
8127 | sched_extend_bb (); | |
8128 | } | |
8129 | ||
496d7bb0 MK |
8130 | /* Initialize BEFORE_RECOVERY variable. */ |
8131 | static void | |
e855c69d | 8132 | init_before_recovery (basic_block *before_recovery_ptr) |
496d7bb0 MK |
8133 | { |
8134 | basic_block last; | |
8135 | edge e; | |
8136 | ||
fefa31b5 | 8137 | last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb; |
0fd4b31d | 8138 | e = find_fallthru_edge_from (last); |
496d7bb0 MK |
8139 | |
8140 | if (e) | |
8141 | { | |
b8698a0f | 8142 | /* We create two basic blocks: |
496d7bb0 | 8143 | 1. Single instruction block is inserted right after E->SRC |
b8698a0f | 8144 | and has jump to |
496d7bb0 MK |
8145 | 2. Empty block right before EXIT_BLOCK. |
8146 | Between these two blocks recovery blocks will be emitted. */ | |
8147 | ||
8148 | basic_block single, empty; | |
496d7bb0 | 8149 | |
b8698a0f | 8150 | /* If the fallthrough edge to exit we've found is from the block we've |
e855c69d AB |
8151 | created before, don't do anything more. */ |
8152 | if (last == after_recovery) | |
8153 | return; | |
496d7bb0 | 8154 | |
e855c69d AB |
8155 | adding_bb_to_current_region_p = false; |
8156 | ||
8157 | single = sched_create_empty_bb (last); | |
8158 | empty = sched_create_empty_bb (single); | |
8159 | ||
8160 | /* Add new blocks to the root loop. */ | |
8161 | if (current_loops != NULL) | |
8162 | { | |
9771b263 DN |
8163 | add_bb_to_loop (single, (*current_loops->larray)[0]); |
8164 | add_bb_to_loop (empty, (*current_loops->larray)[0]); | |
e855c69d AB |
8165 | } |
8166 | ||
8167 | single->count = last->count; | |
496d7bb0 | 8168 | empty->count = last->count; |
496d7bb0 MK |
8169 | BB_COPY_PARTITION (single, last); |
8170 | BB_COPY_PARTITION (empty, last); | |
8171 | ||
8172 | redirect_edge_succ (e, single); | |
8173 | make_single_succ_edge (single, empty, 0); | |
fefa31b5 DM |
8174 | make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun), |
8175 | EDGE_FALLTHRU); | |
496d7bb0 | 8176 | |
e67d1102 | 8177 | rtx_code_label *label = block_label (empty); |
ec4a505f | 8178 | rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label), |
e67d1102 | 8179 | BB_END (single)); |
496d7bb0 MK |
8180 | JUMP_LABEL (x) = label; |
8181 | LABEL_NUSES (label)++; | |
e855c69d | 8182 | haifa_init_insn (x); |
b8698a0f | 8183 | |
496d7bb0 MK |
8184 | emit_barrier_after (x); |
8185 | ||
e855c69d AB |
8186 | sched_init_only_bb (empty, NULL); |
8187 | sched_init_only_bb (single, NULL); | |
8188 | sched_extend_bb (); | |
496d7bb0 | 8189 | |
e855c69d | 8190 | adding_bb_to_current_region_p = true; |
496d7bb0 | 8191 | before_recovery = single; |
e855c69d AB |
8192 | after_recovery = empty; |
8193 | ||
8194 | if (before_recovery_ptr) | |
8195 | *before_recovery_ptr = before_recovery; | |
496d7bb0 MK |
8196 | |
8197 | if (sched_verbose >= 2 && spec_info->dump) | |
8198 | fprintf (spec_info->dump, | |
b8698a0f L |
8199 | ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n", |
8200 | last->index, single->index, empty->index); | |
496d7bb0 MK |
8201 | } |
8202 | else | |
8203 | before_recovery = last; | |
8204 | } | |
8205 | ||
8206 | /* Returns new recovery block. */ | |
e855c69d AB |
8207 | basic_block |
8208 | sched_create_recovery_block (basic_block *before_recovery_ptr) | |
496d7bb0 | 8209 | { |
9b517712 | 8210 | rtx_insn *barrier; |
496d7bb0 | 8211 | basic_block rec; |
b8698a0f | 8212 | |
e2f6ff94 MK |
8213 | haifa_recovery_bb_recently_added_p = true; |
8214 | haifa_recovery_bb_ever_added_p = true; | |
496d7bb0 | 8215 | |
e855c69d | 8216 | init_before_recovery (before_recovery_ptr); |
496d7bb0 | 8217 | |
96370780 MK |
8218 | barrier = get_last_bb_insn (before_recovery); |
8219 | gcc_assert (BARRIER_P (barrier)); | |
8220 | ||
e67d1102 | 8221 | rtx_insn *label = emit_label_after (gen_label_rtx (), barrier); |
96370780 MK |
8222 | |
8223 | rec = create_basic_block (label, label, before_recovery); | |
8224 | ||
e855c69d | 8225 | /* A recovery block always ends with an unconditional jump. */ |
496d7bb0 MK |
8226 | emit_barrier_after (BB_END (rec)); |
8227 | ||
8228 | if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED) | |
8229 | BB_SET_PARTITION (rec, BB_COLD_PARTITION); | |
b8698a0f L |
8230 | |
8231 | if (sched_verbose && spec_info->dump) | |
496d7bb0 MK |
8232 | fprintf (spec_info->dump, ";;\t\tGenerated recovery block rec%d\n", |
8233 | rec->index); | |
8234 | ||
496d7bb0 MK |
8235 | return rec; |
8236 | } | |
8237 | ||
e855c69d AB |
8238 | /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB |
8239 | and emit necessary jumps. */ | |
8240 | void | |
8241 | sched_create_recovery_edges (basic_block first_bb, basic_block rec, | |
8242 | basic_block second_bb) | |
8243 | { | |
e855c69d AB |
8244 | int edge_flags; |
8245 | ||
8246 | /* This is fixing of incoming edge. */ | |
b8698a0f | 8247 | /* ??? Which other flags should be specified? */ |
e855c69d AB |
8248 | if (BB_PARTITION (first_bb) != BB_PARTITION (rec)) |
8249 | /* Partition type is the same, if it is "unpartitioned". */ | |
8250 | edge_flags = EDGE_CROSSING; | |
8251 | else | |
8252 | edge_flags = 0; | |
b8698a0f | 8253 | |
cea768b0 JH |
8254 | edge e2 = single_succ_edge (first_bb); |
8255 | edge e = make_edge (first_bb, rec, edge_flags); | |
8256 | ||
8257 | /* TODO: The actual probability can be determined and is computed as | |
8258 | 'todo_spec' variable in create_check_block_twin and | |
8259 | in sel-sched.c `check_ds' in create_speculation_check. */ | |
8260 | e->probability = profile_probability::very_unlikely (); | |
ef30ab83 | 8261 | rec->count = e->count (); |
cea768b0 | 8262 | e2->probability = e->probability.invert (); |
cea768b0 | 8263 | |
e67d1102 | 8264 | rtx_code_label *label = block_label (second_bb); |
ec4a505f RS |
8265 | rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label), |
8266 | BB_END (rec)); | |
e855c69d AB |
8267 | JUMP_LABEL (jump) = label; |
8268 | LABEL_NUSES (label)++; | |
8269 | ||
8270 | if (BB_PARTITION (second_bb) != BB_PARTITION (rec)) | |
8271 | /* Partition type is the same, if it is "unpartitioned". */ | |
8272 | { | |
8273 | /* Rewritten from cfgrtl.c. */ | |
0b6bc904 | 8274 | if (crtl->has_bb_partition && targetm_common.have_named_sections) |
e855c69d | 8275 | { |
efc0b2bd ILT |
8276 | /* We don't need the same note for the check because |
8277 | any_condjump_p (check) == true. */ | |
339ba33b | 8278 | CROSSING_JUMP_P (jump) = 1; |
e855c69d AB |
8279 | } |
8280 | edge_flags = EDGE_CROSSING; | |
8281 | } | |
8282 | else | |
b8698a0f | 8283 | edge_flags = 0; |
e855c69d | 8284 | |
b8698a0f | 8285 | make_single_succ_edge (rec, second_bb, edge_flags); |
00c4e97c AB |
8286 | if (dom_info_available_p (CDI_DOMINATORS)) |
8287 | set_immediate_dominator (CDI_DOMINATORS, rec, first_bb); | |
e855c69d AB |
8288 | } |
8289 | ||
496d7bb0 MK |
8290 | /* This function creates recovery code for INSN. If MUTATE_P is nonzero, |
8291 | INSN is a simple check, that should be converted to branchy one. */ | |
8292 | static void | |
ce1ce33a | 8293 | create_check_block_twin (rtx_insn *insn, bool mutate_p) |
496d7bb0 MK |
8294 | { |
8295 | basic_block rec; | |
ce1ce33a DM |
8296 | rtx_insn *label, *check, *twin; |
8297 | rtx check_pat; | |
496d7bb0 | 8298 | ds_t fs; |
e2f6ff94 MK |
8299 | sd_iterator_def sd_it; |
8300 | dep_t dep; | |
8301 | dep_def _new_dep, *new_dep = &_new_dep; | |
e855c69d | 8302 | ds_t todo_spec; |
496d7bb0 | 8303 | |
e855c69d AB |
8304 | gcc_assert (ORIG_PAT (insn) != NULL_RTX); |
8305 | ||
8306 | if (!mutate_p) | |
8307 | todo_spec = TODO_SPEC (insn); | |
8308 | else | |
8309 | { | |
8310 | gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn) | |
8311 | && (TODO_SPEC (insn) & SPECULATIVE) == 0); | |
8312 | ||
8313 | todo_spec = CHECK_SPEC (insn); | |
8314 | } | |
8315 | ||
8316 | todo_spec &= SPECULATIVE; | |
496d7bb0 MK |
8317 | |
8318 | /* Create recovery block. */ | |
388092d5 | 8319 | if (mutate_p || targetm.sched.needs_block_p (todo_spec)) |
496d7bb0 | 8320 | { |
e855c69d | 8321 | rec = sched_create_recovery_block (NULL); |
496d7bb0 MK |
8322 | label = BB_HEAD (rec); |
8323 | } | |
8324 | else | |
8325 | { | |
fefa31b5 | 8326 | rec = EXIT_BLOCK_PTR_FOR_FN (cfun); |
ce1ce33a | 8327 | label = NULL; |
496d7bb0 MK |
8328 | } |
8329 | ||
8330 | /* Emit CHECK. */ | |
ce1ce33a | 8331 | check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec); |
496d7bb0 | 8332 | |
fefa31b5 | 8333 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8334 | { |
8335 | /* To have mem_reg alive at the beginning of second_bb, | |
b8698a0f | 8336 | we emit check BEFORE insn, so insn after splitting |
496d7bb0 MK |
8337 | insn will be at the beginning of second_bb, which will |
8338 | provide us with the correct life information. */ | |
ce1ce33a | 8339 | check = emit_jump_insn_before (check_pat, insn); |
496d7bb0 MK |
8340 | JUMP_LABEL (check) = label; |
8341 | LABEL_NUSES (label)++; | |
8342 | } | |
8343 | else | |
ce1ce33a | 8344 | check = emit_insn_before (check_pat, insn); |
496d7bb0 MK |
8345 | |
8346 | /* Extend data structures. */ | |
e855c69d AB |
8347 | haifa_init_insn (check); |
8348 | ||
8349 | /* CHECK is being added to current region. Extend ready list. */ | |
8350 | gcc_assert (sched_ready_n_insns != -1); | |
8351 | sched_extend_ready_list (sched_ready_n_insns + 1); | |
8352 | ||
8353 | if (current_sched_info->add_remove_insn) | |
8354 | current_sched_info->add_remove_insn (insn, 0); | |
8355 | ||
496d7bb0 MK |
8356 | RECOVERY_BLOCK (check) = rec; |
8357 | ||
8358 | if (sched_verbose && spec_info->dump) | |
8359 | fprintf (spec_info->dump, ";;\t\tGenerated check insn : %s\n", | |
8360 | (*current_sched_info->print_insn) (check, 0)); | |
8361 | ||
8362 | gcc_assert (ORIG_PAT (insn)); | |
8363 | ||
917f1b7e | 8364 | /* Initialize TWIN (twin is a duplicate of original instruction |
496d7bb0 | 8365 | in the recovery block). */ |
fefa31b5 | 8366 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 | 8367 | { |
e2f6ff94 MK |
8368 | sd_iterator_def sd_it; |
8369 | dep_t dep; | |
8370 | ||
8371 | FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep) | |
8372 | if ((DEP_STATUS (dep) & DEP_OUTPUT) != 0) | |
496d7bb0 | 8373 | { |
e2f6ff94 | 8374 | struct _dep _dep2, *dep2 = &_dep2; |
b198261f | 8375 | |
e2f6ff94 | 8376 | init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE); |
b198261f | 8377 | |
e2f6ff94 | 8378 | sd_add_dep (dep2, true); |
496d7bb0 MK |
8379 | } |
8380 | ||
8381 | twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec)); | |
e855c69d | 8382 | haifa_init_insn (twin); |
496d7bb0 MK |
8383 | |
8384 | if (sched_verbose && spec_info->dump) | |
8385 | /* INSN_BB (insn) isn't determined for twin insns yet. | |
8386 | So we can't use current_sched_info->print_insn. */ | |
8387 | fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n", | |
8388 | INSN_UID (twin), rec->index); | |
8389 | } | |
8390 | else | |
8391 | { | |
8392 | ORIG_PAT (check) = ORIG_PAT (insn); | |
8393 | HAS_INTERNAL_DEP (check) = 1; | |
8394 | twin = check; | |
8395 | /* ??? We probably should change all OUTPUT dependencies to | |
8396 | (TRUE | OUTPUT). */ | |
8397 | } | |
8398 | ||
e2f6ff94 MK |
8399 | /* Copy all resolved back dependencies of INSN to TWIN. This will |
8400 | provide correct value for INSN_TICK (TWIN). */ | |
8401 | sd_copy_back_deps (twin, insn, true); | |
496d7bb0 | 8402 | |
fefa31b5 | 8403 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8404 | /* In case of branchy check, fix CFG. */ |
8405 | { | |
8406 | basic_block first_bb, second_bb; | |
f57aa6b0 | 8407 | rtx_insn *jump; |
496d7bb0 MK |
8408 | |
8409 | first_bb = BLOCK_FOR_INSN (check); | |
e855c69d | 8410 | second_bb = sched_split_block (first_bb, check); |
496d7bb0 | 8411 | |
e855c69d | 8412 | sched_create_recovery_edges (first_bb, rec, second_bb); |
496d7bb0 | 8413 | |
b8698a0f | 8414 | sched_init_only_bb (second_bb, first_bb); |
fefa31b5 | 8415 | sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun)); |
e855c69d AB |
8416 | |
8417 | jump = BB_END (rec); | |
8418 | haifa_init_insn (jump); | |
496d7bb0 MK |
8419 | } |
8420 | ||
b8698a0f | 8421 | /* Move backward dependences from INSN to CHECK and |
496d7bb0 | 8422 | move forward dependences from INSN to TWIN. */ |
e2f6ff94 MK |
8423 | |
8424 | /* First, create dependencies between INSN's producers and CHECK & TWIN. */ | |
8425 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
496d7bb0 | 8426 | { |
9b517712 | 8427 | rtx_insn *pro = DEP_PRO (dep); |
496d7bb0 MK |
8428 | ds_t ds; |
8429 | ||
8430 | /* If BEGIN_DATA: [insn ~~TRUE~~> producer]: | |
8431 | check --TRUE--> producer ??? or ANTI ??? | |
8432 | twin --TRUE--> producer | |
8433 | twin --ANTI--> check | |
b8698a0f | 8434 | |
496d7bb0 MK |
8435 | If BEGIN_CONTROL: [insn ~~ANTI~~> producer]: |
8436 | check --ANTI--> producer | |
8437 | twin --ANTI--> producer | |
8438 | twin --ANTI--> check | |
8439 | ||
8440 | If BE_IN_SPEC: [insn ~~TRUE~~> producer]: | |
8441 | check ~~TRUE~~> producer | |
8442 | twin ~~TRUE~~> producer | |
b8698a0f | 8443 | twin --ANTI--> check */ |
496d7bb0 | 8444 | |
e2f6ff94 | 8445 | ds = DEP_STATUS (dep); |
496d7bb0 MK |
8446 | |
8447 | if (ds & BEGIN_SPEC) | |
8448 | { | |
8449 | gcc_assert (!mutate_p); | |
8450 | ds &= ~BEGIN_SPEC; | |
8451 | } | |
8452 | ||
e2f6ff94 MK |
8453 | init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds); |
8454 | sd_add_dep (new_dep, false); | |
8455 | ||
fefa31b5 | 8456 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 | 8457 | { |
0e0ce50d | 8458 | DEP_CON (new_dep) = twin; |
e2f6ff94 | 8459 | sd_add_dep (new_dep, false); |
b8698a0f | 8460 | } |
496d7bb0 MK |
8461 | } |
8462 | ||
e2f6ff94 MK |
8463 | /* Second, remove backward dependencies of INSN. */ |
8464 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
8465 | sd_iterator_cond (&sd_it, &dep);) | |
8466 | { | |
8467 | if ((DEP_STATUS (dep) & BEGIN_SPEC) | |
8468 | || mutate_p) | |
8469 | /* We can delete this dep because we overcome it with | |
8470 | BEGIN_SPECULATION. */ | |
8471 | sd_delete_dep (sd_it); | |
8472 | else | |
8473 | sd_iterator_next (&sd_it); | |
8474 | } | |
496d7bb0 | 8475 | |
e2f6ff94 | 8476 | /* Future Speculations. Determine what BE_IN speculations will be like. */ |
496d7bb0 MK |
8477 | fs = 0; |
8478 | ||
8479 | /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only | |
8480 | here. */ | |
b8698a0f | 8481 | |
496d7bb0 | 8482 | gcc_assert (!DONE_SPEC (insn)); |
b8698a0f | 8483 | |
496d7bb0 | 8484 | if (!mutate_p) |
b8698a0f | 8485 | { |
496d7bb0 MK |
8486 | ds_t ts = TODO_SPEC (insn); |
8487 | ||
8488 | DONE_SPEC (insn) = ts & BEGIN_SPEC; | |
8489 | CHECK_SPEC (check) = ts & BEGIN_SPEC; | |
8490 | ||
15dc95cb | 8491 | /* Luckiness of future speculations solely depends upon initial |
e2f6ff94 | 8492 | BEGIN speculation. */ |
496d7bb0 MK |
8493 | if (ts & BEGIN_DATA) |
8494 | fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA)); | |
8495 | if (ts & BEGIN_CONTROL) | |
e2f6ff94 MK |
8496 | fs = set_dep_weak (fs, BE_IN_CONTROL, |
8497 | get_dep_weak (ts, BEGIN_CONTROL)); | |
496d7bb0 MK |
8498 | } |
8499 | else | |
8500 | CHECK_SPEC (check) = CHECK_SPEC (insn); | |
8501 | ||
8502 | /* Future speculations: call the helper. */ | |
e2f6ff94 | 8503 | process_insn_forw_deps_be_in_spec (insn, twin, fs); |
496d7bb0 | 8504 | |
fefa31b5 | 8505 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8506 | { |
8507 | /* Which types of dependencies should we use here is, | |
8508 | generally, machine-dependent question... But, for now, | |
8509 | it is not. */ | |
8510 | ||
8511 | if (!mutate_p) | |
8512 | { | |
e2f6ff94 MK |
8513 | init_dep (new_dep, insn, check, REG_DEP_TRUE); |
8514 | sd_add_dep (new_dep, false); | |
8515 | ||
8516 | init_dep (new_dep, insn, twin, REG_DEP_OUTPUT); | |
8517 | sd_add_dep (new_dep, false); | |
496d7bb0 MK |
8518 | } |
8519 | else | |
8520 | { | |
b8698a0f | 8521 | if (spec_info->dump) |
496d7bb0 MK |
8522 | fprintf (spec_info->dump, ";;\t\tRemoved simple check : %s\n", |
8523 | (*current_sched_info->print_insn) (insn, 0)); | |
8524 | ||
e2f6ff94 MK |
8525 | /* Remove all dependencies of the INSN. */ |
8526 | { | |
8527 | sd_it = sd_iterator_start (insn, (SD_LIST_FORW | |
8528 | | SD_LIST_BACK | |
8529 | | SD_LIST_RES_BACK)); | |
8530 | while (sd_iterator_cond (&sd_it, &dep)) | |
8531 | sd_delete_dep (sd_it); | |
8532 | } | |
496d7bb0 | 8533 | |
e2f6ff94 MK |
8534 | /* If former check (INSN) already was moved to the ready (or queue) |
8535 | list, add new check (CHECK) there too. */ | |
496d7bb0 MK |
8536 | if (QUEUE_INDEX (insn) != QUEUE_NOWHERE) |
8537 | try_ready (check); | |
8538 | ||
e2f6ff94 MK |
8539 | /* Remove old check from instruction stream and free its |
8540 | data. */ | |
496d7bb0 MK |
8541 | sched_remove_insn (insn); |
8542 | } | |
8543 | ||
e2f6ff94 MK |
8544 | init_dep (new_dep, check, twin, REG_DEP_ANTI); |
8545 | sd_add_dep (new_dep, false); | |
496d7bb0 MK |
8546 | } |
8547 | else | |
e2f6ff94 MK |
8548 | { |
8549 | init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE | DEP_OUTPUT); | |
8550 | sd_add_dep (new_dep, false); | |
8551 | } | |
496d7bb0 MK |
8552 | |
8553 | if (!mutate_p) | |
917f1b7e | 8554 | /* Fix priorities. If MUTATE_P is nonzero, this is not necessary, |
496d7bb0 MK |
8555 | because it'll be done later in add_to_speculative_block. */ |
8556 | { | |
8c681247 | 8557 | auto_vec<rtx_insn *> priorities_roots; |
916fa4f0 MK |
8558 | |
8559 | clear_priorities (twin, &priorities_roots); | |
8560 | calc_priorities (priorities_roots); | |
496d7bb0 MK |
8561 | } |
8562 | } | |
8563 | ||
8564 | /* Removes dependency between instructions in the recovery block REC | |
8565 | and usual region instructions. It keeps inner dependences so it | |
917f1b7e | 8566 | won't be necessary to recompute them. */ |
496d7bb0 MK |
8567 | static void |
8568 | fix_recovery_deps (basic_block rec) | |
8569 | { | |
9b517712 | 8570 | rtx_insn *note, *insn, *jump; |
9cad8a67 | 8571 | auto_vec<rtx_insn *, 10> ready_list; |
8f9b31f7 | 8572 | auto_bitmap in_ready; |
b8698a0f | 8573 | |
496d7bb0 MK |
8574 | /* NOTE - a basic block note. */ |
8575 | note = NEXT_INSN (BB_HEAD (rec)); | |
8576 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); | |
8577 | insn = BB_END (rec); | |
8578 | gcc_assert (JUMP_P (insn)); | |
8579 | insn = PREV_INSN (insn); | |
8580 | ||
8581 | do | |
e2f6ff94 MK |
8582 | { |
8583 | sd_iterator_def sd_it; | |
8584 | dep_t dep; | |
496d7bb0 | 8585 | |
e2f6ff94 MK |
8586 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); |
8587 | sd_iterator_cond (&sd_it, &dep);) | |
8588 | { | |
9b517712 | 8589 | rtx_insn *consumer = DEP_CON (dep); |
496d7bb0 MK |
8590 | |
8591 | if (BLOCK_FOR_INSN (consumer) != rec) | |
8592 | { | |
e2f6ff94 | 8593 | sd_delete_dep (sd_it); |
496d7bb0 | 8594 | |
8f9b31f7 | 8595 | if (bitmap_set_bit (in_ready, INSN_LUID (consumer))) |
9cad8a67 | 8596 | ready_list.safe_push (consumer); |
496d7bb0 MK |
8597 | } |
8598 | else | |
8599 | { | |
e2f6ff94 | 8600 | gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE); |
496d7bb0 | 8601 | |
e2f6ff94 | 8602 | sd_iterator_next (&sd_it); |
496d7bb0 MK |
8603 | } |
8604 | } | |
b8698a0f | 8605 | |
496d7bb0 MK |
8606 | insn = PREV_INSN (insn); |
8607 | } | |
8608 | while (insn != note); | |
8609 | ||
496d7bb0 | 8610 | /* Try to add instructions to the ready or queue list. */ |
9cad8a67 TS |
8611 | unsigned int i; |
8612 | rtx_insn *temp; | |
8613 | FOR_EACH_VEC_ELT_REVERSE (ready_list, i, temp) | |
8614 | try_ready (temp); | |
496d7bb0 MK |
8615 | |
8616 | /* Fixing jump's dependences. */ | |
8617 | insn = BB_HEAD (rec); | |
8618 | jump = BB_END (rec); | |
b8698a0f | 8619 | |
496d7bb0 MK |
8620 | gcc_assert (LABEL_P (insn)); |
8621 | insn = NEXT_INSN (insn); | |
b8698a0f | 8622 | |
496d7bb0 MK |
8623 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn)); |
8624 | add_jump_dependencies (insn, jump); | |
8625 | } | |
8626 | ||
e2724e63 BS |
8627 | /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa |
8628 | instruction data. */ | |
8629 | static bool | |
f57aa6b0 | 8630 | haifa_change_pattern (rtx_insn *insn, rtx new_pat) |
496d7bb0 MK |
8631 | { |
8632 | int t; | |
8633 | ||
8634 | t = validate_change (insn, &PATTERN (insn), new_pat, 0); | |
e2724e63 BS |
8635 | if (!t) |
8636 | return false; | |
e855c69d | 8637 | |
1a83e602 | 8638 | update_insn_after_change (insn); |
e2724e63 | 8639 | return true; |
496d7bb0 MK |
8640 | } |
8641 | ||
496d7bb0 MK |
8642 | /* -1 - can't speculate, |
8643 | 0 - for speculation with REQUEST mode it is OK to use | |
8644 | current instruction pattern, | |
8645 | 1 - need to change pattern for *NEW_PAT to be speculative. */ | |
e855c69d | 8646 | int |
ac44248e | 8647 | sched_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat) |
496d7bb0 MK |
8648 | { |
8649 | gcc_assert (current_sched_info->flags & DO_SPECULATION | |
e2f6ff94 MK |
8650 | && (request & SPECULATIVE) |
8651 | && sched_insn_is_legitimate_for_speculation_p (insn, request)); | |
496d7bb0 | 8652 | |
e2f6ff94 | 8653 | if ((request & spec_info->mask) != request) |
496d7bb0 | 8654 | return -1; |
496d7bb0 | 8655 | |
e2f6ff94 MK |
8656 | if (request & BE_IN_SPEC |
8657 | && !(request & BEGIN_SPEC)) | |
8658 | return 0; | |
496d7bb0 | 8659 | |
e855c69d AB |
8660 | return targetm.sched.speculate_insn (insn, request, new_pat); |
8661 | } | |
8662 | ||
8663 | static int | |
f57aa6b0 | 8664 | haifa_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat) |
e855c69d AB |
8665 | { |
8666 | gcc_assert (sched_deps_info->generate_spec_deps | |
8667 | && !IS_SPECULATION_CHECK_P (insn)); | |
8668 | ||
8669 | if (HAS_INTERNAL_DEP (insn) | |
8670 | || SCHED_GROUP_P (insn)) | |
8671 | return -1; | |
8672 | ||
8673 | return sched_speculate_insn (insn, request, new_pat); | |
496d7bb0 MK |
8674 | } |
8675 | ||
8676 | /* Print some information about block BB, which starts with HEAD and | |
8677 | ends with TAIL, before scheduling it. | |
8678 | I is zero, if scheduler is about to start with the fresh ebb. */ | |
8679 | static void | |
f57aa6b0 | 8680 | dump_new_block_header (int i, basic_block bb, rtx_insn *head, rtx_insn *tail) |
496d7bb0 MK |
8681 | { |
8682 | if (!i) | |
8683 | fprintf (sched_dump, | |
8684 | ";; ======================================================\n"); | |
8685 | else | |
8686 | fprintf (sched_dump, | |
8687 | ";; =====================ADVANCING TO=====================\n"); | |
8688 | fprintf (sched_dump, | |
8689 | ";; -- basic block %d from %d to %d -- %s reload\n", | |
8690 | bb->index, INSN_UID (head), INSN_UID (tail), | |
8691 | (reload_completed ? "after" : "before")); | |
8692 | fprintf (sched_dump, | |
8693 | ";; ======================================================\n"); | |
8694 | fprintf (sched_dump, "\n"); | |
8695 | } | |
8696 | ||
8697 | /* Unlink basic block notes and labels and saves them, so they | |
8698 | can be easily restored. We unlink basic block notes in EBB to | |
917f1b7e | 8699 | provide back-compatibility with the previous code, as target backends |
496d7bb0 MK |
8700 | assume, that there'll be only instructions between |
8701 | current_sched_info->{head and tail}. We restore these notes as soon | |
8702 | as we can. | |
8703 | FIRST (LAST) is the first (last) basic block in the ebb. | |
8704 | NB: In usual case (FIRST == LAST) nothing is really done. */ | |
8705 | void | |
8706 | unlink_bb_notes (basic_block first, basic_block last) | |
8707 | { | |
8708 | /* We DON'T unlink basic block notes of the first block in the ebb. */ | |
8709 | if (first == last) | |
8710 | return; | |
8711 | ||
9b517712 | 8712 | bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun)); |
496d7bb0 MK |
8713 | |
8714 | /* Make a sentinel. */ | |
fefa31b5 | 8715 | if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8716 | bb_header[last->next_bb->index] = 0; |
8717 | ||
8718 | first = first->next_bb; | |
8719 | do | |
8720 | { | |
9b517712 | 8721 | rtx_insn *prev, *label, *note, *next; |
496d7bb0 MK |
8722 | |
8723 | label = BB_HEAD (last); | |
8724 | if (LABEL_P (label)) | |
8725 | note = NEXT_INSN (label); | |
8726 | else | |
b8698a0f | 8727 | note = label; |
496d7bb0 MK |
8728 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
8729 | ||
8730 | prev = PREV_INSN (label); | |
8731 | next = NEXT_INSN (note); | |
8732 | gcc_assert (prev && next); | |
8733 | ||
0f82e5c9 DM |
8734 | SET_NEXT_INSN (prev) = next; |
8735 | SET_PREV_INSN (next) = prev; | |
496d7bb0 MK |
8736 | |
8737 | bb_header[last->index] = label; | |
8738 | ||
8739 | if (last == first) | |
8740 | break; | |
b8698a0f | 8741 | |
496d7bb0 MK |
8742 | last = last->prev_bb; |
8743 | } | |
8744 | while (1); | |
8745 | } | |
8746 | ||
8747 | /* Restore basic block notes. | |
8748 | FIRST is the first basic block in the ebb. */ | |
8749 | static void | |
8750 | restore_bb_notes (basic_block first) | |
8751 | { | |
8752 | if (!bb_header) | |
8753 | return; | |
8754 | ||
8755 | /* We DON'T unlink basic block notes of the first block in the ebb. */ | |
b8698a0f | 8756 | first = first->next_bb; |
496d7bb0 MK |
8757 | /* Remember: FIRST is actually a second basic block in the ebb. */ |
8758 | ||
fefa31b5 | 8759 | while (first != EXIT_BLOCK_PTR_FOR_FN (cfun) |
496d7bb0 MK |
8760 | && bb_header[first->index]) |
8761 | { | |
9b517712 | 8762 | rtx_insn *prev, *label, *note, *next; |
b8698a0f | 8763 | |
496d7bb0 MK |
8764 | label = bb_header[first->index]; |
8765 | prev = PREV_INSN (label); | |
8766 | next = NEXT_INSN (prev); | |
8767 | ||
8768 | if (LABEL_P (label)) | |
8769 | note = NEXT_INSN (label); | |
8770 | else | |
b8698a0f | 8771 | note = label; |
496d7bb0 MK |
8772 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
8773 | ||
8774 | bb_header[first->index] = 0; | |
8775 | ||
0f82e5c9 DM |
8776 | SET_NEXT_INSN (prev) = label; |
8777 | SET_NEXT_INSN (note) = next; | |
8778 | SET_PREV_INSN (next) = note; | |
b8698a0f | 8779 | |
496d7bb0 MK |
8780 | first = first->next_bb; |
8781 | } | |
8782 | ||
8783 | free (bb_header); | |
8784 | bb_header = 0; | |
8785 | } | |
8786 | ||
496d7bb0 MK |
8787 | /* Helper function. |
8788 | Fix CFG after both in- and inter-block movement of | |
8789 | control_flow_insn_p JUMP. */ | |
8790 | static void | |
f57aa6b0 | 8791 | fix_jump_move (rtx_insn *jump) |
496d7bb0 MK |
8792 | { |
8793 | basic_block bb, jump_bb, jump_bb_next; | |
8794 | ||
8795 | bb = BLOCK_FOR_INSN (PREV_INSN (jump)); | |
8796 | jump_bb = BLOCK_FOR_INSN (jump); | |
8797 | jump_bb_next = jump_bb->next_bb; | |
8798 | ||
e855c69d | 8799 | gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS |
d7bfd907 | 8800 | || IS_SPECULATION_BRANCHY_CHECK_P (jump)); |
b8698a0f | 8801 | |
496d7bb0 MK |
8802 | if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next))) |
8803 | /* if jump_bb_next is not empty. */ | |
1130d5e3 | 8804 | BB_END (jump_bb) = BB_END (jump_bb_next); |
496d7bb0 MK |
8805 | |
8806 | if (BB_END (bb) != PREV_INSN (jump)) | |
8807 | /* Then there are instruction after jump that should be placed | |
8808 | to jump_bb_next. */ | |
1130d5e3 | 8809 | BB_END (jump_bb_next) = BB_END (bb); |
496d7bb0 MK |
8810 | else |
8811 | /* Otherwise jump_bb_next is empty. */ | |
1130d5e3 | 8812 | BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next)); |
496d7bb0 MK |
8813 | |
8814 | /* To make assertion in move_insn happy. */ | |
1130d5e3 | 8815 | BB_END (bb) = PREV_INSN (jump); |
496d7bb0 MK |
8816 | |
8817 | update_bb_for_insn (jump_bb_next); | |
8818 | } | |
8819 | ||
8820 | /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */ | |
8821 | static void | |
f57aa6b0 | 8822 | move_block_after_check (rtx_insn *jump) |
496d7bb0 MK |
8823 | { |
8824 | basic_block bb, jump_bb, jump_bb_next; | |
9771b263 | 8825 | vec<edge, va_gc> *t; |
496d7bb0 MK |
8826 | |
8827 | bb = BLOCK_FOR_INSN (PREV_INSN (jump)); | |
8828 | jump_bb = BLOCK_FOR_INSN (jump); | |
8829 | jump_bb_next = jump_bb->next_bb; | |
b8698a0f | 8830 | |
496d7bb0 | 8831 | update_bb_for_insn (jump_bb); |
b8698a0f | 8832 | |
d7bfd907 MK |
8833 | gcc_assert (IS_SPECULATION_CHECK_P (jump) |
8834 | || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next))); | |
496d7bb0 MK |
8835 | |
8836 | unlink_block (jump_bb_next); | |
8837 | link_block (jump_bb_next, bb); | |
8838 | ||
8839 | t = bb->succs; | |
8840 | bb->succs = 0; | |
8841 | move_succs (&(jump_bb->succs), bb); | |
8842 | move_succs (&(jump_bb_next->succs), jump_bb); | |
8843 | move_succs (&t, jump_bb_next); | |
6fb5fa3c DB |
8844 | |
8845 | df_mark_solutions_dirty (); | |
b8698a0f | 8846 | |
e855c69d AB |
8847 | common_sched_info->fix_recovery_cfg |
8848 | (bb->index, jump_bb->index, jump_bb_next->index); | |
496d7bb0 MK |
8849 | } |
8850 | ||
8851 | /* Helper function for move_block_after_check. | |
8852 | This functions attaches edge vector pointed to by SUCCSP to | |
8853 | block TO. */ | |
8854 | static void | |
9771b263 | 8855 | move_succs (vec<edge, va_gc> **succsp, basic_block to) |
496d7bb0 MK |
8856 | { |
8857 | edge e; | |
8858 | edge_iterator ei; | |
8859 | ||
8860 | gcc_assert (to->succs == 0); | |
8861 | ||
8862 | to->succs = *succsp; | |
8863 | ||
8864 | FOR_EACH_EDGE (e, ei, to->succs) | |
8865 | e->src = to; | |
8866 | ||
8867 | *succsp = 0; | |
8868 | } | |
8869 | ||
496d7bb0 MK |
8870 | /* Remove INSN from the instruction stream. |
8871 | INSN should have any dependencies. */ | |
8872 | static void | |
ce1ce33a | 8873 | sched_remove_insn (rtx_insn *insn) |
496d7bb0 | 8874 | { |
e2f6ff94 MK |
8875 | sd_finish_insn (insn); |
8876 | ||
496d7bb0 MK |
8877 | change_queue_index (insn, QUEUE_NOWHERE); |
8878 | current_sched_info->add_remove_insn (insn, 1); | |
1f397f45 | 8879 | delete_insn (insn); |
496d7bb0 MK |
8880 | } |
8881 | ||
916fa4f0 MK |
8882 | /* Clear priorities of all instructions, that are forward dependent on INSN. |
8883 | Store in vector pointed to by ROOTS_PTR insns on which priority () should | |
8884 | be invoked to initialize all cleared priorities. */ | |
496d7bb0 | 8885 | static void |
6144a836 | 8886 | clear_priorities (rtx_insn *insn, rtx_vec_t *roots_ptr) |
496d7bb0 | 8887 | { |
e2f6ff94 MK |
8888 | sd_iterator_def sd_it; |
8889 | dep_t dep; | |
916fa4f0 MK |
8890 | bool insn_is_root_p = true; |
8891 | ||
8892 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED); | |
496d7bb0 | 8893 | |
e2f6ff94 | 8894 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) |
496d7bb0 | 8895 | { |
9b517712 | 8896 | rtx_insn *pro = DEP_PRO (dep); |
496d7bb0 | 8897 | |
916fa4f0 MK |
8898 | if (INSN_PRIORITY_STATUS (pro) >= 0 |
8899 | && QUEUE_INDEX (insn) != QUEUE_SCHEDULED) | |
496d7bb0 | 8900 | { |
916fa4f0 MK |
8901 | /* If DEP doesn't contribute to priority then INSN itself should |
8902 | be added to priority roots. */ | |
8903 | if (contributes_to_priority_p (dep)) | |
8904 | insn_is_root_p = false; | |
8905 | ||
8906 | INSN_PRIORITY_STATUS (pro) = -1; | |
8907 | clear_priorities (pro, roots_ptr); | |
496d7bb0 MK |
8908 | } |
8909 | } | |
916fa4f0 MK |
8910 | |
8911 | if (insn_is_root_p) | |
9771b263 | 8912 | roots_ptr->safe_push (insn); |
496d7bb0 MK |
8913 | } |
8914 | ||
8915 | /* Recompute priorities of instructions, whose priorities might have been | |
916fa4f0 MK |
8916 | changed. ROOTS is a vector of instructions whose priority computation will |
8917 | trigger initialization of all cleared priorities. */ | |
496d7bb0 | 8918 | static void |
916fa4f0 | 8919 | calc_priorities (rtx_vec_t roots) |
496d7bb0 | 8920 | { |
916fa4f0 | 8921 | int i; |
6144a836 | 8922 | rtx_insn *insn; |
496d7bb0 | 8923 | |
9771b263 | 8924 | FOR_EACH_VEC_ELT (roots, i, insn) |
916fa4f0 | 8925 | priority (insn); |
496d7bb0 MK |
8926 | } |
8927 | ||
8928 | ||
8929 | /* Add dependences between JUMP and other instructions in the recovery | |
8930 | block. INSN is the first insn the recovery block. */ | |
8931 | static void | |
0e0ce50d | 8932 | add_jump_dependencies (rtx_insn *insn, rtx_insn *jump) |
496d7bb0 MK |
8933 | { |
8934 | do | |
8935 | { | |
8936 | insn = NEXT_INSN (insn); | |
8937 | if (insn == jump) | |
8938 | break; | |
b8698a0f | 8939 | |
7a1aca9c | 8940 | if (dep_list_size (insn, SD_LIST_FORW) == 0) |
e2f6ff94 MK |
8941 | { |
8942 | dep_def _new_dep, *new_dep = &_new_dep; | |
8943 | ||
8944 | init_dep (new_dep, insn, jump, REG_DEP_ANTI); | |
8945 | sd_add_dep (new_dep, false); | |
8946 | } | |
496d7bb0 MK |
8947 | } |
8948 | while (1); | |
b198261f | 8949 | |
e2f6ff94 | 8950 | gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK)); |
496d7bb0 MK |
8951 | } |
8952 | ||
e855c69d | 8953 | /* Extend data structures for logical insn UID. */ |
a95b23b4 BS |
8954 | void |
8955 | sched_extend_luids (void) | |
e855c69d AB |
8956 | { |
8957 | int new_luids_max_uid = get_max_uid () + 1; | |
8958 | ||
9771b263 | 8959 | sched_luids.safe_grow_cleared (new_luids_max_uid); |
e855c69d AB |
8960 | } |
8961 | ||
8962 | /* Initialize LUID for INSN. */ | |
a95b23b4 | 8963 | void |
f57aa6b0 | 8964 | sched_init_insn_luid (rtx_insn *insn) |
e855c69d AB |
8965 | { |
8966 | int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn); | |
8967 | int luid; | |
8968 | ||
8969 | if (i >= 0) | |
8970 | { | |
8971 | luid = sched_max_luid; | |
8972 | sched_max_luid += i; | |
8973 | } | |
8974 | else | |
8975 | luid = -1; | |
8976 | ||
8977 | SET_INSN_LUID (insn, luid); | |
8978 | } | |
8979 | ||
a95b23b4 | 8980 | /* Initialize luids for BBS. |
e855c69d AB |
8981 | The hook common_sched_info->luid_for_non_insn () is used to determine |
8982 | if notes, labels, etc. need luids. */ | |
8983 | void | |
a95b23b4 | 8984 | sched_init_luids (bb_vec_t bbs) |
e855c69d | 8985 | { |
a95b23b4 BS |
8986 | int i; |
8987 | basic_block bb; | |
8988 | ||
8989 | sched_extend_luids (); | |
9771b263 | 8990 | FOR_EACH_VEC_ELT (bbs, i, bb) |
e855c69d | 8991 | { |
f57aa6b0 | 8992 | rtx_insn *insn; |
e855c69d | 8993 | |
a95b23b4 BS |
8994 | FOR_BB_INSNS (bb, insn) |
8995 | sched_init_insn_luid (insn); | |
8996 | } | |
e855c69d AB |
8997 | } |
8998 | ||
8999 | /* Free LUIDs. */ | |
9000 | void | |
9001 | sched_finish_luids (void) | |
9002 | { | |
9771b263 | 9003 | sched_luids.release (); |
e855c69d AB |
9004 | sched_max_luid = 1; |
9005 | } | |
9006 | ||
9007 | /* Return logical uid of INSN. Helpful while debugging. */ | |
9008 | int | |
f57aa6b0 | 9009 | insn_luid (rtx_insn *insn) |
e855c69d AB |
9010 | { |
9011 | return INSN_LUID (insn); | |
9012 | } | |
9013 | ||
9014 | /* Extend per insn data in the target. */ | |
9015 | void | |
9016 | sched_extend_target (void) | |
9017 | { | |
9018 | if (targetm.sched.h_i_d_extended) | |
9019 | targetm.sched.h_i_d_extended (); | |
9020 | } | |
9021 | ||
9022 | /* Extend global scheduler structures (those, that live across calls to | |
9023 | schedule_block) to include information about just emitted INSN. */ | |
9024 | static void | |
9025 | extend_h_i_d (void) | |
9026 | { | |
9771b263 | 9027 | int reserve = (get_max_uid () + 1 - h_i_d.length ()); |
b8698a0f | 9028 | if (reserve > 0 |
9771b263 | 9029 | && ! h_i_d.space (reserve)) |
e855c69d | 9030 | { |
9771b263 | 9031 | h_i_d.safe_grow_cleared (3 * get_max_uid () / 2); |
e855c69d AB |
9032 | sched_extend_target (); |
9033 | } | |
9034 | } | |
9035 | ||
9036 | /* Initialize h_i_d entry of the INSN with default values. | |
9037 | Values, that are not explicitly initialized here, hold zero. */ | |
9038 | static void | |
f57aa6b0 | 9039 | init_h_i_d (rtx_insn *insn) |
e855c69d AB |
9040 | { |
9041 | if (INSN_LUID (insn) > 0) | |
9042 | { | |
9043 | INSN_COST (insn) = -1; | |
e855c69d AB |
9044 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; |
9045 | INSN_TICK (insn) = INVALID_TICK; | |
26965010 | 9046 | INSN_EXACT_TICK (insn) = INVALID_TICK; |
e855c69d AB |
9047 | INTER_TICK (insn) = INVALID_TICK; |
9048 | TODO_SPEC (insn) = HARD_DEP; | |
340c7904 MK |
9049 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status |
9050 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
9051 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status | |
9052 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
e855c69d AB |
9053 | } |
9054 | } | |
9055 | ||
a95b23b4 | 9056 | /* Initialize haifa_insn_data for BBS. */ |
e855c69d | 9057 | void |
a95b23b4 | 9058 | haifa_init_h_i_d (bb_vec_t bbs) |
e855c69d | 9059 | { |
a95b23b4 BS |
9060 | int i; |
9061 | basic_block bb; | |
9062 | ||
9063 | extend_h_i_d (); | |
9771b263 | 9064 | FOR_EACH_VEC_ELT (bbs, i, bb) |
e855c69d | 9065 | { |
f57aa6b0 | 9066 | rtx_insn *insn; |
e855c69d | 9067 | |
a95b23b4 BS |
9068 | FOR_BB_INSNS (bb, insn) |
9069 | init_h_i_d (insn); | |
9070 | } | |
e855c69d AB |
9071 | } |
9072 | ||
9073 | /* Finalize haifa_insn_data. */ | |
9074 | void | |
9075 | haifa_finish_h_i_d (void) | |
9076 | { | |
ce18efcb VM |
9077 | int i; |
9078 | haifa_insn_data_t data; | |
36fb33de ML |
9079 | reg_use_data *use, *next_use; |
9080 | reg_set_data *set, *next_set; | |
ce18efcb | 9081 | |
9771b263 | 9082 | FOR_EACH_VEC_ELT (h_i_d, i, data) |
ce18efcb | 9083 | { |
7a1aca9c | 9084 | free (data->max_reg_pressure); |
04695783 | 9085 | free (data->reg_pressure); |
36fb33de | 9086 | for (use = data->reg_use_list; use != NULL; use = next_use) |
ce18efcb | 9087 | { |
36fb33de | 9088 | next_use = use->next_insn_use; |
ce18efcb VM |
9089 | free (use); |
9090 | } | |
36fb33de ML |
9091 | for (set = data->reg_set_list; set != NULL; set = next_set) |
9092 | { | |
9093 | next_set = set->next_insn_set; | |
9094 | free (set); | |
9095 | } | |
9096 | ||
ce18efcb | 9097 | } |
9771b263 | 9098 | h_i_d.release (); |
e855c69d AB |
9099 | } |
9100 | ||
9101 | /* Init data for the new insn INSN. */ | |
9102 | static void | |
f57aa6b0 | 9103 | haifa_init_insn (rtx_insn *insn) |
e855c69d AB |
9104 | { |
9105 | gcc_assert (insn != NULL); | |
9106 | ||
a95b23b4 BS |
9107 | sched_extend_luids (); |
9108 | sched_init_insn_luid (insn); | |
e855c69d AB |
9109 | sched_extend_target (); |
9110 | sched_deps_init (false); | |
a95b23b4 BS |
9111 | extend_h_i_d (); |
9112 | init_h_i_d (insn); | |
e855c69d AB |
9113 | |
9114 | if (adding_bb_to_current_region_p) | |
9115 | { | |
9116 | sd_init_insn (insn); | |
9117 | ||
9118 | /* Extend dependency caches by one element. */ | |
9119 | extend_dependency_caches (1, false); | |
9120 | } | |
7a1aca9c | 9121 | if (sched_pressure != SCHED_PRESSURE_NONE) |
19ac7892 | 9122 | init_insn_reg_pressure_info (insn); |
e855c69d AB |
9123 | } |
9124 | ||
9125 | /* Init data for the new basic block BB which comes after AFTER. */ | |
9126 | static void | |
9127 | haifa_init_only_bb (basic_block bb, basic_block after) | |
9128 | { | |
9129 | gcc_assert (bb != NULL); | |
9130 | ||
9131 | sched_init_bbs (); | |
9132 | ||
9133 | if (common_sched_info->add_block) | |
9134 | /* This changes only data structures of the front-end. */ | |
9135 | common_sched_info->add_block (bb, after); | |
9136 | } | |
9137 | ||
9138 | /* A generic version of sched_split_block (). */ | |
9139 | basic_block | |
9140 | sched_split_block_1 (basic_block first_bb, rtx after) | |
9141 | { | |
9142 | edge e; | |
9143 | ||
9144 | e = split_block (first_bb, after); | |
9145 | gcc_assert (e->src == first_bb); | |
9146 | ||
b8698a0f | 9147 | /* sched_split_block emits note if *check == BB_END. Probably it |
e855c69d AB |
9148 | is better to rip that note off. */ |
9149 | ||
9150 | return e->dest; | |
9151 | } | |
9152 | ||
9153 | /* A generic version of sched_create_empty_bb (). */ | |
9154 | basic_block | |
9155 | sched_create_empty_bb_1 (basic_block after) | |
9156 | { | |
9157 | return create_empty_bb (after); | |
9158 | } | |
9159 | ||
9dcc2e87 TS |
9160 | /* Insert PAT as an INSN into the schedule and update the necessary data |
9161 | structures to account for it. */ | |
ce1ce33a | 9162 | rtx_insn * |
9dcc2e87 TS |
9163 | sched_emit_insn (rtx pat) |
9164 | { | |
ce1ce33a | 9165 | rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ()); |
9dcc2e87 | 9166 | haifa_init_insn (insn); |
2dfdcb4b UW |
9167 | |
9168 | if (current_sched_info->add_remove_insn) | |
9169 | current_sched_info->add_remove_insn (insn, 0); | |
9170 | ||
9171 | (*current_sched_info->begin_schedule_ready) (insn); | |
9771b263 | 9172 | scheduled_insns.safe_push (insn); |
2dfdcb4b UW |
9173 | |
9174 | last_scheduled_insn = insn; | |
9dcc2e87 TS |
9175 | return insn; |
9176 | } | |
9177 | ||
7942e47e RY |
9178 | /* This function returns a candidate satisfying dispatch constraints from |
9179 | the ready list. */ | |
9180 | ||
ce1ce33a | 9181 | static rtx_insn * |
7942e47e RY |
9182 | ready_remove_first_dispatch (struct ready_list *ready) |
9183 | { | |
9184 | int i; | |
ce1ce33a | 9185 | rtx_insn *insn = ready_element (ready, 0); |
7942e47e RY |
9186 | |
9187 | if (ready->n_ready == 1 | |
7942e47e | 9188 | || !INSN_P (insn) |
d7b6661b | 9189 | || INSN_CODE (insn) < 0 |
7942e47e RY |
9190 | || !active_insn_p (insn) |
9191 | || targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW)) | |
9192 | return ready_remove_first (ready); | |
9193 | ||
9194 | for (i = 1; i < ready->n_ready; i++) | |
9195 | { | |
9196 | insn = ready_element (ready, i); | |
9197 | ||
d7b6661b SB |
9198 | if (!INSN_P (insn) |
9199 | || INSN_CODE (insn) < 0 | |
7942e47e RY |
9200 | || !active_insn_p (insn)) |
9201 | continue; | |
9202 | ||
9203 | if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW)) | |
9204 | { | |
9205 | /* Return ith element of ready. */ | |
9206 | insn = ready_remove (ready, i); | |
9207 | return insn; | |
9208 | } | |
9209 | } | |
9210 | ||
ac44248e | 9211 | if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION)) |
7942e47e RY |
9212 | return ready_remove_first (ready); |
9213 | ||
9214 | for (i = 1; i < ready->n_ready; i++) | |
9215 | { | |
9216 | insn = ready_element (ready, i); | |
9217 | ||
98eaa66f | 9218 | if (!INSN_P (insn) |
d7b6661b | 9219 | || INSN_CODE (insn) < 0 |
7942e47e RY |
9220 | || !active_insn_p (insn)) |
9221 | continue; | |
9222 | ||
9223 | /* Return i-th element of ready. */ | |
9224 | if (targetm.sched.dispatch (insn, IS_CMP)) | |
9225 | return ready_remove (ready, i); | |
9226 | } | |
9227 | ||
9228 | return ready_remove_first (ready); | |
9229 | } | |
9230 | ||
9231 | /* Get number of ready insn in the ready list. */ | |
9232 | ||
9233 | int | |
9234 | number_in_ready (void) | |
9235 | { | |
9236 | return ready.n_ready; | |
9237 | } | |
9238 | ||
9239 | /* Get number of ready's in the ready list. */ | |
9240 | ||
6a263466 | 9241 | rtx_insn * |
7942e47e RY |
9242 | get_ready_element (int i) |
9243 | { | |
9244 | return ready_element (&ready, i); | |
9245 | } | |
9246 | ||
8c660648 | 9247 | #endif /* INSN_SCHEDULING */ |