This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
Re: [OpenACC 7/11] execution model
- From: Nathan Sidwell <nathan at acm dot org>
- To: GCC Patches <gcc-patches at gcc dot gnu dot org>
- Cc: Jakub Jelinek <jakub at redhat dot com>, Bernd Schmidt <bschmidt at redhat dot com>, Jason Merrill <jason at redhat dot com>, "Joseph S. Myers" <joseph at codesourcery dot com>
- Date: Wed, 21 Oct 2015 15:42:26 -0400
- Subject: Re: [OpenACC 7/11] execution model
- Authentication-results: sourceware.org; auth=none
- References: <5627DD78 dot 9040302 at acm dot org>
This patch is the early lowering part of OpenACC loops. Rather than piggy-back
onto expand_omp_for_static_nochunk & expand_omp_for_static_chunk, we have a new
function 'expand_oacc_for', which does the OpenACC equivalent expension. Except
that it uses a new internal builtin to abstract the actual loop step, intial
value, bound and chunking. We end up turning a loop of the form
for (i = b; i < e; i += s)
{...}
into
chunk=0
num_chunks = IFN_LOOP (CHUNKS, ...);
step = IFN_LOOP (STEP, ...);
head:
offset = IFN_LOOP (OFFSET, ...);
bound = IFN_LOOP (BOUND, ...);
if (!(offset < bound)) goto bottom;
body:
i = offset + b;
{...}
offset += step;
if (offset < bound) goto body;
bottom:
chunk++;
if (chunk < num_chunks) goto head;
In addition to marking up the loop itself like that, we also emit partitioning
markers around the loop. The whole of the above sequence will be wrapped with:
IFN_UNIQUE (HEAD_MARK, <partitioning flags>)
IFN_UNIQUE (FORK) // repeat for each potential axis
IFN_UNIQUE (HEAD_MARK)
<loop here>
IFN_UNIQUE (TAIL_MARK, ...)
IFN_UNIQUE (JOIN) // repeat for each potential axis
IFN_UNIQUE (TAIL_MARK)
The reason for the rather heavy weight head and tail marker series is for
reductions. Those will insert more internal fns just before and just after the
fork and join functions.
The fork and join pattern is duplicated for each potential axis, but does not
specify which axis -- that is all determined in the device-specific oacc
lowering pass.
The patch introduces 4 variants of the IFN_UNIQUE function along with a new
IFN_LOOP function. I see I also included IFN_OACC_DIM_POS and IFN_OACC_DIM_SIZE
which provide the size of a compute axis and a position along it. Those are
actually used in the next patch and not here.
nathan
2015-10-20 Nathan Sidwell <nathan@codesourcery.com>
* omp-low.c (struct omp_context): Remove gwv_below, gwv_this
fields.
(enum oacc_loop_flags): New.
(enclosing_target_ctx): May return NULL.
(ctx_in_oacc_kernels_region): New.
(is_oacc_parallel, is_oaccc_kernels): New.
(check_oacc_kernel_gwv): New.
(oacc_loop_or_target_p): Delete.
(scan_omp_for): Don't calculate gwv mask. Check parallel clause
operands. Strip reductions fro kernels.
(scan_omp_target): Don't calculate gwv mask.
(lower_oacc_head_mark, lower_oacc_loop_marker,
lower_oacc_head_tail): New.
(expand_omp_for_static_nochunk, expand_omp_for_static_chunk):
Remove OpenACC handling.
(struct oacc_collapse): New.
(expand_oacc_collapse_init, expand_oacc_collapse_vars): New.
(expand_oacc_for): New.
(expand_omp_for): Call expand_oacc_for.
(lower_omp_for): Call lower_oacc_head_tail.
* internal-fn.def (IFN_UNIQUE_OACC_FORK, IFN_UNIQUE_OACC_JOIN,
IFN_OACC_HEAD_MARK, IFN_OACC_TAIL_MARK): Define.
(IFN_GOACC_DIM_SIZE, IFN_GOACC_DIM_POS): New.
(IFN_GOACC_LOOP): New.
(IFN_GOACC_LOOP_CHUNKS, IFN_GOACC_LOOP_STEP,
IFN_GOACC_LOOP_OFFSET, IFN_GOACC_LOOP_BOUND): New.
* internal-fn.c (expand_UNIQUE): Handle IFN_UNIQUE_OACC_FORK,
IFN_UNIQUE_OACC_JOIN.
(expand_GOACC_DIM_SIZE, expand_GOACC_DIM_POS, expand_GOACC_LOOP): New.
Index: gcc/omp-low.c
===================================================================
--- gcc/omp-low.c (revision 229101)
+++ gcc/omp-low.c (working copy)
@@ -199,14 +200,6 @@ struct omp_context
/* True if this construct can be cancelled. */
bool cancellable;
-
- /* For OpenACC loops, a mask of gang, worker and vector used at
- levels below this one. */
- int gwv_below;
- /* For OpenACC loops, a mask of gang, worker and vector used at
- this level and above. For parallel and kernels clauses, a mask
- indicating which of num_gangs/num_workers/num_vectors was used. */
- int gwv_this;
};
/* A structure holding the elements of:
@@ -233,6 +226,24 @@ struct omp_for_data
struct omp_for_data_loop *loops;
};
+/* Flags for an OpenACC loop. */
+
+enum oacc_loop_flags
+ {
+ OLF_SEQ = 1u << 0, /* Explicitly sequential */
+ OLF_AUTO = 1u << 1, /* Compiler chooses axes. */
+ OLF_INDEPENDENT = 1u << 2, /* Iterations are known independent. */
+ OLF_GANG_STATIC = 1u << 3, /* Gang partitioning is static (has op). */
+
+ /* Explicitly specified loop axes. */
+ OLF_DIM_BASE = 4,
+ OLF_DIM_GANG = 1u << (OLF_DIM_BASE + GOMP_DIM_GANG),
+ OLF_DIM_WORKER = 1u << (OLF_DIM_BASE + GOMP_DIM_WORKER),
+ OLF_DIM_VECTOR = 1u << (OLF_DIM_BASE + GOMP_DIM_VECTOR),
+
+ OLF_MAX = OLF_DIM_BASE + GOMP_DIM_MAX
+ };
+
static splay_tree all_contexts;
static int taskreg_nesting_level;
@@ -255,6 +292,28 @@ static gphi *find_phi_with_arg_on_edge (
*handled_ops_p = false; \
break;
+/* Return true if CTX corresponds to an oacc parallel region. */
+
+static bool
+is_oacc_parallel (omp_context *ctx)
+{
+ enum gimple_code outer_type = gimple_code (ctx->stmt);
+ return ((outer_type == GIMPLE_OMP_TARGET)
+ && (gimple_omp_target_kind (ctx->stmt)
+ == GF_OMP_TARGET_KIND_OACC_PARALLEL));
+}
+
+/* Return true if CTX corresponds to an oacc kernels region. */
+
+static bool
+is_oacc_kernels (omp_context *ctx)
+{
+ enum gimple_code outer_type = gimple_code (ctx->stmt);
+ return ((outer_type == GIMPLE_OMP_TARGET)
+ && (gimple_omp_target_kind (ctx->stmt)
+ == GF_OMP_TARGET_KIND_OACC_KERNELS));
+}
+
/* Helper function to get the name of the array containing the partial
reductions for OpenACC reductions. */
static const char *
@@ -2889,28 +2948,92 @@ finish_taskreg_scan (omp_context *ctx)
}
}
+/* Find the enclosing offload context. */
static omp_context *
enclosing_target_ctx (omp_context *ctx)
{
- while (ctx != NULL
- && gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
- ctx = ctx->outer;
- gcc_assert (ctx != NULL);
+ for (; ctx; ctx = ctx->outer)
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
+ break;
+
return ctx;
}
+/* Return true if ctx is part of an oacc kernels region. */
+
static bool
-oacc_loop_or_target_p (gimple *stmt)
+ctx_in_oacc_kernels_region (omp_context *ctx)
+{
+ for (;ctx != NULL; ctx = ctx->outer)
+ {
+ gimple *stmt = ctx->stmt;
+ if (gimple_code (stmt) == GIMPLE_OMP_TARGET
+ && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
+ return true;
+ }
+
+ return false;
+}
+
+/* Check the parallelism clauses inside a kernels regions.
+ Until kernels handling moves to use the same loop indirection
+ scheme as parallel, we need to do this checking early. */
+
+static unsigned
+check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
{
- enum gimple_code outer_type = gimple_code (stmt);
- return ((outer_type == GIMPLE_OMP_TARGET
- && ((gimple_omp_target_kind (stmt)
- == GF_OMP_TARGET_KIND_OACC_PARALLEL)
- || (gimple_omp_target_kind (stmt)
- == GF_OMP_TARGET_KIND_OACC_KERNELS)))
- || (outer_type == GIMPLE_OMP_FOR
- && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP));
+ bool checking = true;
+ unsigned outer_mask = 0;
+ unsigned this_mask = 0;
+ bool has_seq = false, has_auto = false;
+
+ if (ctx->outer)
+ outer_mask = check_oacc_kernel_gwv (NULL, ctx->outer);
+ if (!stmt)
+ {
+ checking = false;
+ if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
+ return outer_mask;
+ stmt = as_a <gomp_for *> (ctx->stmt);
+ }
+
+ for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
+ {
+ switch (OMP_CLAUSE_CODE (c))
+ {
+ case OMP_CLAUSE_GANG:
+ this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
+ break;
+ case OMP_CLAUSE_WORKER:
+ this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
+ break;
+ case OMP_CLAUSE_VECTOR:
+ this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
+ break;
+ case OMP_CLAUSE_SEQ:
+ has_seq = true;
+ break;
+ case OMP_CLAUSE_AUTO:
+ has_auto = true;
+ break;
+ default:
+ break;
+ }
+ }
+
+ if (checking)
+ {
+ if (has_seq && (this_mask || has_auto))
+ error_at (gimple_location (stmt), "%<seq%> overrides other OpenACC loop specifiers");
+ else if (has_auto && this_mask)
+ error_at (gimple_location (stmt), "%<auto%> conflicts with other OpenACC loop specifiers");
+
+ if (this_mask & outer_mask)
+ error_at (gimple_location (stmt), "inner loop uses same OpenACC parallelism as containing loop");
+ }
+
+ return outer_mask | this_mask;
}
/* Scan a GIMPLE_OMP_FOR. */
@@ -2918,52 +3041,62 @@ oacc_loop_or_target_p (gimple *stmt)
static void
scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
{
- enum gimple_code outer_type = GIMPLE_ERROR_MARK;
omp_context *ctx;
size_t i;
tree clauses = gimple_omp_for_clauses (stmt);
- if (outer_ctx)
- outer_type = gimple_code (outer_ctx->stmt);
-
ctx = new_omp_context (stmt, outer_ctx);
if (is_gimple_omp_oacc (stmt))
{
- if (outer_ctx && outer_type == GIMPLE_OMP_FOR)
- ctx->gwv_this = outer_ctx->gwv_this;
- for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
- {
- int val;
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_GANG)
- val = MASK_GANG;
- else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WORKER)
- val = MASK_WORKER;
- else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR)
- val = MASK_VECTOR;
- else
- continue;
- ctx->gwv_this |= val;
- if (!outer_ctx)
- {
- /* Skip; not nested inside a region. */
- continue;
- }
- if (!oacc_loop_or_target_p (outer_ctx->stmt))
+ omp_context *tgt = enclosing_target_ctx (outer_ctx);
+
+ if (!tgt || is_oacc_parallel (tgt))
+ for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+ {
+ char const *check = NULL;
+
+ switch (OMP_CLAUSE_CODE (c))
+ {
+ case OMP_CLAUSE_GANG:
+ check = "gang";
+ break;
+
+ case OMP_CLAUSE_WORKER:
+ check = "worker";
+ break;
+
+ case OMP_CLAUSE_VECTOR:
+ check = "vector";
+ break;
+
+ default:
+ break;
+ }
+
+ if (check && OMP_CLAUSE_OPERAND (c, 0))
+ error_at (gimple_location (stmt),
+ "argument not permitted on %<%s%> clause in"
+ " OpenACC %<parallel%>", check);
+ }
+
+ if (tgt && is_oacc_kernels (tgt))
+ {
+ /* Strip out reductions, as they are not handled yet. */
+ tree *prev_ptr = &clauses;
+
+ while (tree probe = *prev_ptr)
{
- /* Skip; not nested inside an OpenACC region. */
- continue;
- }
- if (outer_type == GIMPLE_OMP_FOR)
- outer_ctx->gwv_below |= val;
- if (OMP_CLAUSE_OPERAND (c, 0) != NULL_TREE)
- {
- omp_context *enclosing = enclosing_target_ctx (outer_ctx);
- if (gimple_omp_target_kind (enclosing->stmt)
- == GF_OMP_TARGET_KIND_OACC_PARALLEL)
- error_at (gimple_location (stmt),
- "no arguments allowed to gang, worker and vector clauses inside parallel");
+ tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
+
+ if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
+ *prev_ptr = *next_ptr;
+ else
+ prev_ptr = next_ptr;
}
+
+ gimple_omp_for_set_clauses (stmt, clauses);
+ check_oacc_kernel_gwv (stmt, ctx);
}
}
@@ -2978,19 +3111,6 @@ scan_omp_for (gomp_for *stmt, omp_contex
scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
}
scan_omp (gimple_omp_body_ptr (stmt), ctx);
-
- if (is_gimple_omp_oacc (stmt))
- {
- if (ctx->gwv_this & ctx->gwv_below)
- error_at (gimple_location (stmt),
- "gang, worker and vector may occur only once in a loop nest");
- else if (ctx->gwv_below != 0
- && ctx->gwv_this > ctx->gwv_below)
- error_at (gimple_location (stmt),
- "gang, worker and vector must occur in this order in a loop nest");
- if (outer_ctx && outer_type == GIMPLE_OMP_FOR)
- outer_ctx->gwv_below |= ctx->gwv_below;
- }
}
/* Scan an OpenMP sections directive. */
@@ -3061,19 +3181,6 @@ scan_omp_target (gomp_target *stmt, omp_
gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
}
- if (is_gimple_omp_oacc (stmt))
- {
- for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
- {
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_GANGS)
- ctx->gwv_this |= MASK_GANG;
- else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_WORKERS)
- ctx->gwv_this |= MASK_WORKER;
- else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR_LENGTH)
- ctx->gwv_this |= MASK_VECTOR;
- }
- }
-
scan_sharing_clauses (clauses, ctx);
scan_omp (gimple_omp_body_ptr (stmt), ctx);
@@ -5769,6 +5885,166 @@ lower_send_shared_vars (gimple_seq *ilis
}
}
+/* Emit an OpenACC head marker call, encapulating the partitioning and
+ other information that must be processed by the target compiler.
+ Return the maximum number of dimensions the associated loop might
+ be partitioned over. */
+
+static unsigned
+lower_oacc_head_mark (location_t loc, tree clauses,
+ gimple_seq *seq, omp_context *ctx)
+{
+ unsigned levels = 0;
+ unsigned tag = 0;
+ tree gang_static = NULL_TREE;
+ auto_vec<tree, 1> args;
+
+ args.quick_push (build_int_cst
+ (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
+ for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+ {
+ switch (OMP_CLAUSE_CODE (c))
+ {
+ case OMP_CLAUSE_GANG:
+ tag |= OLF_DIM_GANG;
+ gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
+ /* static:* is represented by -1, and we can ignore it, as
+ scheduling is always static. */
+ if (gang_static && integer_minus_onep (gang_static))
+ gang_static = NULL_TREE;
+ levels++;
+ break;
+
+ case OMP_CLAUSE_WORKER:
+ tag |= OLF_DIM_WORKER;
+ levels++;
+ break;
+
+ case OMP_CLAUSE_VECTOR:
+ tag |= OLF_DIM_VECTOR;
+ levels++;
+ break;
+
+ case OMP_CLAUSE_SEQ:
+ tag |= OLF_SEQ;
+ break;
+
+ case OMP_CLAUSE_AUTO:
+ tag |= OLF_AUTO;
+ break;
+
+ case OMP_CLAUSE_INDEPENDENT:
+ tag |= OLF_INDEPENDENT;
+ break;
+
+ default:
+ continue;
+ }
+ }
+
+ if (gang_static)
+ {
+ if (DECL_P (gang_static))
+ gang_static = build_outer_var_ref (gang_static, ctx);
+ tag |= OLF_GANG_STATIC;
+ }
+
+ /* In a parallel region, loops are implicitly INDEPENDENT. */
+ omp_context *tgt = enclosing_target_ctx (ctx);
+ if (!tgt || is_oacc_parallel (tgt))
+ tag |= OLF_INDEPENDENT;
+
+ /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
+ if (!(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1) << OLF_DIM_BASE)
+ | OLF_SEQ)))
+ tag |= OLF_AUTO;
+
+ /* Ensure at least one level. */
+ if (!levels)
+ levels++;
+
+ args.safe_push (build_int_cst (integer_type_node, levels));
+ args.safe_push (build_int_cst (integer_type_node, tag));
+ if (gang_static)
+ args.safe_push (gang_static);
+
+ gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
+ gimple_set_location (call, loc);
+ gimple_seq_add_stmt (seq, call);
+
+ return levels;
+}
+
+/* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
+ partitioning level of the enclosed region. */
+
+static void
+lower_oacc_loop_marker (location_t loc, bool head, tree tofollow,
+ gimple_seq *seq)
+{
+ tree marker = build_int_cst
+ (integer_type_node, (head ? IFN_UNIQUE_OACC_HEAD_MARK
+ : IFN_UNIQUE_OACC_TAIL_MARK));
+ gcall *call = gimple_build_call_internal
+ (IFN_UNIQUE, 1 + (tofollow != NULL_TREE), marker, tofollow);
+ gimple_set_location (call, loc);
+ gimple_seq_add_stmt (seq, call);
+}
+
+/* Generate the before and after OpenACC loop sequences. CLAUSES are
+ the loop clauses, from which we extract reductions. Initialize
+ HEAD and TAIL. */
+
+static void
+lower_oacc_head_tail (location_t loc, tree clauses,
+ gimple_seq *head, gimple_seq *tail, omp_context *ctx)
+{
+ bool inner = false;
+ unsigned count = lower_oacc_head_mark (loc, clauses, head, ctx);
+
+ if (!count)
+ lower_oacc_loop_marker (loc, false, integer_zero_node, tail);
+
+ for (unsigned done = 1; count; count--, done++)
+ {
+ tree place = build_int_cst (integer_type_node, -1);
+ gcall *fork = gimple_build_call_internal
+ (IFN_UNIQUE, 2,
+ build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK), place);
+ gcall *join = gimple_build_call_internal
+ (IFN_UNIQUE, 2,
+ build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN), place);
+ gimple_seq fork_seq = NULL;
+ gimple_seq join_seq = NULL;
+
+ gimple_set_location (fork, loc);
+ gimple_set_location (join, loc);
+
+ /* Mark the beginning of this level sequence. */
+ if (inner)
+ lower_oacc_loop_marker (loc, true,
+ build_int_cst (integer_type_node, count),
+ &fork_seq);
+ lower_oacc_loop_marker (loc, false,
+ build_int_cst (integer_type_node, done),
+ &join_seq);
+
+ gimple_seq_add_stmt (&fork_seq, fork);
+ gimple_seq_add_stmt (&join_seq, join);
+
+ /* Append this level to head. */
+ gimple_seq_add_seq (head, fork_seq);
+ /* Prepend it to tail. */
+ gimple_seq_add_seq (&join_seq, *tail);
+ *tail = join_seq;
+
+ inner = true;
+ }
+
+ /* Mark the end of the sequence. */
+ lower_oacc_loop_marker (loc, true, NULL_TREE, head);
+ lower_oacc_loop_marker (loc, false, NULL_TREE, tail);
+}
/* A convenience function to build an empty GIMPLE_COND with just the
condition. */
@@ -8327,10 +8603,6 @@ expand_omp_for_static_nochunk (struct om
tree *counts = NULL;
tree n1, n2, step;
- gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt)
- != GF_OMP_FOR_KIND_OACC_LOOP)
- || !inner_stmt);
-
itype = type = TREE_TYPE (fd->loop.v);
if (POINTER_TYPE_P (type))
itype = signed_type_for (type);
@@ -8423,10 +8695,6 @@ expand_omp_for_static_nochunk (struct om
nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
break;
- case GF_OMP_FOR_KIND_OACC_LOOP:
- nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
- threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
- break;
default:
gcc_unreachable ();
}
@@ -8653,10 +8921,7 @@ expand_omp_for_static_nochunk (struct om
if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
{
t = gimple_omp_return_lhs (gsi_stmt (gsi));
- if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
- gcc_checking_assert (t == NULL_TREE);
- else
- gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
+ gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
}
gsi_remove (&gsi, true);
@@ -8794,10 +9059,6 @@ expand_omp_for_static_chunk (struct omp_
tree *counts = NULL;
tree n1, n2, step;
- gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt)
- != GF_OMP_FOR_KIND_OACC_LOOP)
- || !inner_stmt);
-
itype = type = TREE_TYPE (fd->loop.v);
if (POINTER_TYPE_P (type))
itype = signed_type_for (type);
@@ -8894,10 +9155,6 @@ expand_omp_for_static_chunk (struct omp_
nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
break;
- case GF_OMP_FOR_KIND_OACC_LOOP:
- nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
- threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
- break;
default:
gcc_unreachable ();
}
@@ -9157,10 +9414,7 @@ expand_omp_for_static_chunk (struct omp_
if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
{
t = gimple_omp_return_lhs (gsi_stmt (gsi));
- if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
- gcc_checking_assert (t == NULL_TREE);
- else
- gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
+ gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
}
gsi_remove (&gsi, true);
@@ -10210,93 +10464,645 @@ expand_omp_taskloop_for_inner (struct om
}
}
-/* Expand the OMP loop defined by REGION. */
+/* Information about members of an OpenACC collapsed loop nest. */
-static void
-expand_omp_for (struct omp_region *region, gimple *inner_stmt)
+struct oacc_collapse
{
- struct omp_for_data fd;
- struct omp_for_data_loop *loops;
+ tree base; /* Base value. */
+ tree iters; /* Number of steps. */
+ tree step; /* step size. */
+};
- loops
- = (struct omp_for_data_loop *)
- alloca (gimple_omp_for_collapse (last_stmt (region->entry))
- * sizeof (struct omp_for_data_loop));
- extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)),
- &fd, loops);
- region->sched_kind = fd.sched_kind;
+/* Helper for expand_oacc_for. Determine collapsed loop information.
+ Fill in COUNTS array. Emit any initialization code before GSI.
+ Return the calculated outer loop bound of BOUND_TYPE. */
- gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
- BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
- FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
- if (region->cont)
- {
- gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
- BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
- FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
- }
- else
- /* If there isn't a continue then this is a degerate case where
- the introduction of abnormal edges during lowering will prevent
- original loops from being detected. Fix that up. */
- loops_state_set (LOOPS_NEED_FIXUP);
+static tree
+expand_oacc_collapse_init (const struct omp_for_data *fd,
+ gimple_stmt_iterator *gsi,
+ oacc_collapse *counts, tree bound_type)
+{
+ tree total = build_int_cst (bound_type, 1);
+ int ix;
+
+ gcc_assert (integer_onep (fd->loop.step));
+ gcc_assert (integer_zerop (fd->loop.n1));
- if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
- expand_omp_simd (region, &fd);
- else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
- expand_cilk_for (region, &fd);
- else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_TASKLOOP)
- {
- if (gimple_omp_for_combined_into_p (fd.for_stmt))
- expand_omp_taskloop_for_inner (region, &fd, inner_stmt);
- else
- expand_omp_taskloop_for_outer (region, &fd, inner_stmt);
- }
- else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
- && !fd.have_ordered)
- {
- if (fd.chunk_size == NULL)
- expand_omp_for_static_nochunk (region, &fd, inner_stmt);
- else
- expand_omp_for_static_chunk (region, &fd, inner_stmt);
- }
- else
+ for (ix = 0; ix != fd->collapse; ix++)
{
- int fn_index, start_ix, next_ix;
+ const omp_for_data_loop *loop = &fd->loops[ix];
- gcc_assert (gimple_omp_for_kind (fd.for_stmt)
- == GF_OMP_FOR_KIND_FOR);
- if (fd.chunk_size == NULL
- && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
- fd.chunk_size = integer_zero_node;
- gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
- fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
- ? 3 : fd.sched_kind;
- if (!fd.ordered)
- fn_index += fd.have_ordered * 4;
- if (fd.ordered)
- start_ix = ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START) + fn_index;
- else
- start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
- next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
- if (fd.iter_type == long_long_unsigned_type_node)
- {
- start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
- - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
- next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
- - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
- }
- expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
- (enum built_in_function) next_ix, inner_stmt);
+ tree iter_type = TREE_TYPE (loop->v);
+ tree diff_type = iter_type;
+ tree plus_type = iter_type;
+
+ gcc_assert (loop->cond_code == fd->loop.cond_code);
+
+ if (POINTER_TYPE_P (iter_type))
+ plus_type = sizetype;
+ if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
+ diff_type = signed_type_for (diff_type);
+
+ tree b = loop->n1;
+ tree e = loop->n2;
+ tree s = loop->step;
+ bool up = loop->cond_code == LT_EXPR;
+ tree dir = build_int_cst (diff_type, up ? +1 : -1);
+ bool negating;
+ tree expr;
+
+ b = force_gimple_operand_gsi (gsi, b, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+ e = force_gimple_operand_gsi (gsi, e, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+
+ /* Convert the step, avoiding possible unsigned->signed overflow. */
+ negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
+ if (negating)
+ s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
+ s = fold_convert (diff_type, s);
+ if (negating)
+ s = fold_build1 (NEGATE_EXPR, diff_type, s);
+ s = force_gimple_operand_gsi (gsi, s, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+
+ /* Determine the range, avoiding possible unsigned->signed overflow. */
+ negating = !up && TYPE_UNSIGNED (iter_type);
+ expr = fold_build2 (MINUS_EXPR, plus_type,
+ fold_convert (plus_type, negating ? b : e),
+ fold_convert (plus_type, negating ? e : b));
+ expr = fold_convert (diff_type, expr);
+ if (negating)
+ expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
+ tree range = force_gimple_operand_gsi
+ (gsi, expr, true, NULL_TREE, true, GSI_SAME_STMT);
+
+ /* Determine number of iterations. */
+ expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
+ expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
+ expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
+
+ tree iters = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+
+ counts[ix].base = b;
+ counts[ix].iters = iters;
+ counts[ix].step = s;
+
+ total = fold_build2 (MULT_EXPR, bound_type, total,
+ fold_convert (bound_type, iters));
}
- if (gimple_in_ssa_p (cfun))
- update_ssa (TODO_update_ssa_only_virtuals);
+ return total;
}
+/* Emit initializers for collapsed loop members. IVAR is the outer
+ loop iteration variable, from which collapsed loop iteration values
+ are calculated. COUNTS array has been initialized by
+ expand_oacc_collapse_inits. */
+
+static void
+expand_oacc_collapse_vars (const struct omp_for_data *fd,
+ gimple_stmt_iterator *gsi,
+ const oacc_collapse *counts, tree ivar)
+{
+ tree ivar_type = TREE_TYPE (ivar);
+
+ /* The most rapidly changing iteration variable is the innermost
+ one. */
+ for (int ix = fd->collapse; ix--;)
+ {
+ const omp_for_data_loop *loop = &fd->loops[ix];
+ const oacc_collapse *collapse = &counts[ix];
+ tree iter_type = TREE_TYPE (loop->v);
+ tree diff_type = TREE_TYPE (collapse->step);
+ tree plus_type = iter_type;
+ enum tree_code plus_code = PLUS_EXPR;
+ tree expr;
+
+ if (POINTER_TYPE_P (iter_type))
+ {
+ plus_code = POINTER_PLUS_EXPR;
+ plus_type = sizetype;
+ }
+
+ expr = build2 (TRUNC_MOD_EXPR, ivar_type, ivar,
+ fold_convert (ivar_type, collapse->iters));
+ expr = build2 (MULT_EXPR, diff_type, fold_convert (diff_type, expr),
+ collapse->step);
+ expr = build2 (plus_code, iter_type, collapse->base,
+ fold_convert (plus_type, expr));
+ expr = force_gimple_operand_gsi (gsi, expr, false, NULL_TREE,
+ true, GSI_SAME_STMT);
+ gassign *ass = gimple_build_assign (loop->v, expr);
+ gsi_insert_before (gsi, ass, GSI_SAME_STMT);
+
+ if (ix)
+ {
+ expr = build2 (TRUNC_DIV_EXPR, ivar_type, ivar,
+ fold_convert (ivar_type, collapse->iters));
+ ivar = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+ }
+ }
+}
+
+/* A subroutine of expand_omp_for. Generate code for an OpenACC
+ partitioned loop. The lowering here is abstracted, in that the
+ loop parameters are passed through internal functions, which are
+ further lowered by oacc_device_lower, once we get to the target
+ compiler. The loop is of the form:
+
+ for (V = B; V LTGT E; V += S) {BODY}
+
+ where LTGT is < or >. We may have a specified chunking size, CHUNKING
+ (constant 0 for no chunking) and we will have a GWV partitioning
+ mask, specifying dimensions over which the loop is to be
+ partitioned (see note below). We generate code that looks like:
+
+ <entry_bb> [incoming FALL->body, BRANCH->exit]
+ typedef signedintify (typeof (V)) T; // underlying signed integral type
+ T range = E - B;
+ T chunk_no = 0;
+ T DIR = LTGT == '<' ? +1 : -1;
+ T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
+ T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
+
+ <head_bb> [created by splitting end of entry_bb]
+ T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
+ T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
+ if (!(offset LTGT bound)) goto bottom_bb;
+
+ <body_bb> [incoming]
+ V = B + offset;
+ {BODY}
+
+ <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
+ offset += step;
+ if (offset LTGT bound) goto body_bb; [*]
+
+ <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
+ chunk_no++;
+ if (chunk < chunk_max) goto head_bb;
+
+ <exit_bb> [incoming]
+ V = B + ((range -/+ 1) / S +/- 1) * S [*]
+
+ [*] Needed if V live at end of loop
+
+ Note: CHUNKING & GWV mask are specified explicitly here. This is a
+ transition, and will be specified by a more general mechanism shortly.
+ */
+
+static void
+expand_oacc_for (struct omp_region *region, struct omp_for_data *fd)
+{
+ tree v = fd->loop.v;
+ enum tree_code cond_code = fd->loop.cond_code;
+ enum tree_code plus_code = PLUS_EXPR;
+
+ tree chunk_size = integer_minus_one_node;
+ tree gwv = integer_zero_node;
+ tree iter_type = TREE_TYPE (v);
+ tree diff_type = iter_type;
+ tree plus_type = iter_type;
+ struct oacc_collapse *counts = NULL;
+
+ gcc_checking_assert (gimple_omp_for_kind (fd->for_stmt)
+ == GF_OMP_FOR_KIND_OACC_LOOP);
+ gcc_assert (!gimple_omp_for_combined_into_p (fd->for_stmt));
+ gcc_assert (cond_code == LT_EXPR || cond_code == GT_EXPR);
+
+ if (POINTER_TYPE_P (iter_type))
+ {
+ plus_code = POINTER_PLUS_EXPR;
+ plus_type = sizetype;
+ }
+ if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
+ diff_type = signed_type_for (diff_type);
+
+ basic_block entry_bb = region->entry; /* BB ending in OMP_FOR */
+ basic_block exit_bb = region->exit; /* BB ending in OMP_RETURN */
+ basic_block cont_bb = region->cont; /* BB ending in OMP_CONTINUE */
+ basic_block bottom_bb = NULL;
+
+ /* entry_bb has two sucessors; the branch edge is to the exit
+ block, fallthrough edge to body. */
+ gcc_assert (EDGE_COUNT (entry_bb->succs) == 2
+ && BRANCH_EDGE (entry_bb)->dest == exit_bb);
+
+ /* If cont_bb non-NULL, it has 2 successors. The branch successor is
+ body_bb, or to a block whose only successor is the body_bb. Its
+ fallthrough successor is the final block (same as the branch
+ successor of the entry_bb). */
+ if (cont_bb)
+ {
+ basic_block body_bb = FALLTHRU_EDGE (entry_bb)->dest;
+ basic_block bed = BRANCH_EDGE (cont_bb)->dest;
+
+ gcc_assert (FALLTHRU_EDGE (cont_bb)->dest == exit_bb);
+ gcc_assert (bed == body_bb || single_succ_edge (bed)->dest == body_bb);
+ }
+ else
+ gcc_assert (!gimple_in_ssa_p (cfun));
+
+ /* The exit block only has entry_bb and cont_bb as predecessors. */
+ gcc_assert (EDGE_COUNT (exit_bb->preds) == 1 + (cont_bb != NULL));
+
+ tree chunk_no;
+ tree chunk_max = NULL_TREE;
+ tree bound, offset;
+ tree step = create_tmp_var (diff_type, ".step");
+ bool up = cond_code == LT_EXPR;
+ tree dir = build_int_cst (diff_type, up ? +1 : -1);
+ bool chunking = !gimple_in_ssa_p (cfun);;
+ bool negating;
+
+ /* SSA instances. */
+ tree offset_incr = NULL_TREE;
+ tree offset_init = NULL_TREE;
+
+ gimple_stmt_iterator gsi;
+ gassign *ass;
+ gcall *call;
+ gimple *stmt;
+ tree expr;
+ location_t loc;
+ edge split, be, fte;
+
+ /* Split the end of entry_bb to create head_bb. */
+ split = split_block (entry_bb, last_stmt (entry_bb));
+ basic_block head_bb = split->dest;
+ entry_bb = split->src;
+
+ /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
+ gsi = gsi_last_bb (entry_bb);
+ gomp_for *for_stmt = as_a <gomp_for *> (gsi_stmt (gsi));
+ loc = gimple_location (for_stmt);
+
+ if (gimple_in_ssa_p (cfun))
+ {
+ offset_init = gimple_omp_for_index (for_stmt, 0);
+ gcc_assert (integer_zerop (fd->loop.n1));
+ /* The SSA parallelizer does gang parallelism. */
+ gwv = build_int_cst (integer_type_node, GOMP_DIM_MASK (GOMP_DIM_GANG));
+ }
+
+ if (fd->collapse > 1)
+ {
+ counts = XALLOCAVEC (struct oacc_collapse, fd->collapse);
+ tree total = expand_oacc_collapse_init (fd, &gsi, counts,
+ TREE_TYPE (fd->loop.n2));
+
+ if (SSA_VAR_P (fd->loop.n2))
+ {
+ total = force_gimple_operand_gsi (&gsi, total, false, NULL_TREE,
+ true, GSI_SAME_STMT);
+ ass = gimple_build_assign (fd->loop.n2, total);
+ gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
+ }
+
+ }
+
+ tree b = fd->loop.n1;
+ tree e = fd->loop.n2;
+ tree s = fd->loop.step;
+
+ b = force_gimple_operand_gsi (&gsi, b, true, NULL_TREE, true, GSI_SAME_STMT);
+ e = force_gimple_operand_gsi (&gsi, e, true, NULL_TREE, true, GSI_SAME_STMT);
+
+ /* Convert the step, avoiding possible unsigned->signed overflow. */
+ negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
+ if (negating)
+ s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
+ s = fold_convert (diff_type, s);
+ if (negating)
+ s = fold_build1 (NEGATE_EXPR, diff_type, s);
+ s = force_gimple_operand_gsi (&gsi, s, true, NULL_TREE, true, GSI_SAME_STMT);
+
+ if (!chunking)
+ chunk_size = integer_zero_node;
+ expr = fold_convert (diff_type, chunk_size);
+ chunk_size = force_gimple_operand_gsi (&gsi, expr, true,
+ NULL_TREE, true, GSI_SAME_STMT);
+ /* Determine the range, avoiding possible unsigned->signed overflow. */
+ negating = !up && TYPE_UNSIGNED (iter_type);
+ expr = fold_build2 (MINUS_EXPR, plus_type,
+ fold_convert (plus_type, negating ? b : e),
+ fold_convert (plus_type, negating ? e : b));
+ expr = fold_convert (diff_type, expr);
+ if (negating)
+ expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
+ tree range = force_gimple_operand_gsi (&gsi, expr, true,
+ NULL_TREE, true, GSI_SAME_STMT);
+
+ chunk_no = build_int_cst (diff_type, 0);
+ if (chunking)
+ {
+ gcc_assert (!gimple_in_ssa_p (cfun));
+
+ expr = chunk_no;
+ chunk_max = create_tmp_var (diff_type, ".chunk_max");
+ chunk_no = create_tmp_var (diff_type, ".chunk_no");
+
+ ass = gimple_build_assign (chunk_no, expr);
+ gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
+
+ call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
+ build_int_cst (integer_type_node,
+ IFN_GOACC_LOOP_CHUNKS),
+ dir, range, s, chunk_size, gwv);
+ gimple_call_set_lhs (call, chunk_max);
+ gimple_set_location (call, loc);
+ gsi_insert_before (&gsi, call, GSI_SAME_STMT);
+ }
+ else
+ chunk_size = chunk_no;
+
+ call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
+ build_int_cst (integer_type_node,
+ IFN_GOACC_LOOP_STEP),
+ dir, range, s, chunk_size, gwv);
+ gimple_call_set_lhs (call, step);
+ gimple_set_location (call, loc);
+ gsi_insert_before (&gsi, call, GSI_SAME_STMT);
+
+ /* Remove the GIMPLE_OMP_FOR. */
+ gsi_remove (&gsi, true);
+
+ /* Fixup edges from head_bb */
+ be = BRANCH_EDGE (head_bb);
+ fte = FALLTHRU_EDGE (head_bb);
+ be->flags |= EDGE_FALSE_VALUE;
+ fte->flags ^= EDGE_FALLTHRU | EDGE_TRUE_VALUE;
+
+ basic_block body_bb = fte->dest;
+
+ if (gimple_in_ssa_p (cfun))
+ {
+ gsi = gsi_last_bb (cont_bb);
+ gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
+
+ offset = gimple_omp_continue_control_use (cont_stmt);
+ offset_incr = gimple_omp_continue_control_def (cont_stmt);
+ }
+ else
+ {
+ offset = create_tmp_var (diff_type, ".offset");
+ offset_init = offset_incr = offset;
+ }
+ bound = create_tmp_var (TREE_TYPE (offset), ".bound");
+
+ /* Loop offset & bound go into head_bb. */
+ gsi = gsi_start_bb (head_bb);
+
+ call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
+ build_int_cst (integer_type_node,
+ IFN_GOACC_LOOP_OFFSET),
+ dir, range, s,
+ chunk_size, gwv, chunk_no);
+ gimple_call_set_lhs (call, offset_init);
+ gimple_set_location (call, loc);
+ gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
+
+ call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
+ build_int_cst (integer_type_node,
+ IFN_GOACC_LOOP_BOUND),
+ dir, range, s,
+ chunk_size, gwv, offset_init);
+ gimple_call_set_lhs (call, bound);
+ gimple_set_location (call, loc);
+ gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
+
+ expr = build2 (cond_code, boolean_type_node, offset_init, bound);
+ gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
+ GSI_CONTINUE_LINKING);
+
+ /* V assignment goes into body_bb. */
+ if (!gimple_in_ssa_p (cfun))
+ {
+ gsi = gsi_start_bb (body_bb);
+
+ expr = build2 (plus_code, iter_type, b,
+ fold_convert (plus_type, offset));
+ expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
+ true, GSI_SAME_STMT);
+ ass = gimple_build_assign (v, expr);
+ gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
+ if (fd->collapse > 1)
+ expand_oacc_collapse_vars (fd, &gsi, counts, v);
+ }
+
+ /* Loop increment goes into cont_bb. If this is not a loop, we
+ will have spawned threads as if it was, and each one will
+ execute one iteration. The specification is not explicit about
+ whether such constructs are ill-formed or not, and they can
+ occur, especially when noreturn routines are involved. */
+ if (cont_bb)
+ {
+ gsi = gsi_last_bb (cont_bb);
+ gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
+ loc = gimple_location (cont_stmt);
+
+ /* Increment offset. */
+ if (gimple_in_ssa_p (cfun))
+ expr= build2 (plus_code, iter_type, offset,
+ fold_convert (plus_type, step));
+ else
+ expr = build2 (PLUS_EXPR, diff_type, offset, step);
+ expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
+ true, GSI_SAME_STMT);
+ ass = gimple_build_assign (offset_incr, expr);
+ gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
+ expr = build2 (cond_code, boolean_type_node, offset_incr, bound);
+ gsi_insert_before (&gsi, gimple_build_cond_empty (expr), GSI_SAME_STMT);
+
+ /* Remove the GIMPLE_OMP_CONTINUE. */
+ gsi_remove (&gsi, true);
+
+ /* Fixup edges from cont_bb */
+ be = BRANCH_EDGE (cont_bb);
+ fte = FALLTHRU_EDGE (cont_bb);
+ be->flags |= EDGE_TRUE_VALUE;
+ fte->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
+
+ if (chunking)
+ {
+ /* Split the beginning of exit_bb to make bottom_bb. We
+ need to insert a nop at the start, because splitting is
+ after a stmt, not before. */
+ gsi = gsi_start_bb (exit_bb);
+ stmt = gimple_build_nop ();
+ gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
+ split = split_block (exit_bb, stmt);
+ bottom_bb = split->src;
+ exit_bb = split->dest;
+ gsi = gsi_last_bb (bottom_bb);
+
+ /* Chunk increment and test goes into bottom_bb. */
+ expr = build2 (PLUS_EXPR, diff_type, chunk_no,
+ build_int_cst (diff_type, 1));
+ ass = gimple_build_assign (chunk_no, expr);
+ gsi_insert_after (&gsi, ass, GSI_CONTINUE_LINKING);
+
+ /* Chunk test at end of bottom_bb. */
+ expr = build2 (LT_EXPR, boolean_type_node, chunk_no, chunk_max);
+ gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
+ GSI_CONTINUE_LINKING);
+
+ /* Fixup edges from bottom_bb. */
+ split->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
+ make_edge (bottom_bb, head_bb, EDGE_TRUE_VALUE);
+ }
+ }
+
+ gsi = gsi_last_bb (exit_bb);
+ gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
+ loc = gimple_location (gsi_stmt (gsi));
+
+ if (!gimple_in_ssa_p (cfun))
+ {
+ /* Insert the final value of V, in case it is live. This is the
+ value for the only thread that survives past the join. */
+ expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
+ expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
+ expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
+ expr = fold_build2 (MULT_EXPR, diff_type, expr, s);
+ expr = build2 (plus_code, iter_type, b, fold_convert (plus_type, expr));
+ expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
+ true, GSI_SAME_STMT);
+ ass = gimple_build_assign (v, expr);
+ gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
+ }
+
+ /* Remove the OMP_RETURN. */
+ gsi_remove (&gsi, true);
+
+ if (cont_bb)
+ {
+ /* We now have one or two nested loops. Update the loop
+ structures. */
+ struct loop *parent = entry_bb->loop_father;
+ struct loop *body = body_bb->loop_father;
+
+ if (chunking)
+ {
+ struct loop *chunk_loop = alloc_loop ();
+ chunk_loop->header = head_bb;
+ chunk_loop->latch = bottom_bb;
+ add_loop (chunk_loop, parent);
+ parent = chunk_loop;
+ }
+ else if (parent != body)
+ {
+ gcc_assert (body->header == body_bb);
+ gcc_assert (body->latch == cont_bb
+ || single_pred (body->latch) == cont_bb);
+ parent = NULL;
+ }
+
+ if (parent)
+ {
+ struct loop *body_loop = alloc_loop ();
+ body_loop->header = body_bb;
+ body_loop->latch = cont_bb;
+ add_loop (body_loop, parent);
+ }
+ }
+}
+
+/* Expand the OMP loop defined by REGION. */
+
+static void
+expand_omp_for (struct omp_region *region, gimple *inner_stmt)
+{
+ struct omp_for_data fd;
+ struct omp_for_data_loop *loops;
+
+ loops
+ = (struct omp_for_data_loop *)
+ alloca (gimple_omp_for_collapse (last_stmt (region->entry))
+ * sizeof (struct omp_for_data_loop));
+ extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)),
+ &fd, loops);
+ region->sched_kind = fd.sched_kind;
+
+ gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
+ BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
+ FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
+ if (region->cont)
+ {
+ gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
+ BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
+ FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
+ }
+ else
+ /* If there isn't a continue then this is a degerate case where
+ the introduction of abnormal edges during lowering will prevent
+ original loops from being detected. Fix that up. */
+ loops_state_set (LOOPS_NEED_FIXUP);
+
+ if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
+ expand_omp_simd (region, &fd);
+ else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
+ expand_cilk_for (region, &fd);
+ else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
+ {
+ gcc_assert (!inner_stmt);
+ expand_oacc_for (region, &fd);
+ }
+ else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_TASKLOOP)
+ {
+ if (gimple_omp_for_combined_into_p (fd.for_stmt))
+ expand_omp_taskloop_for_inner (region, &fd, inner_stmt);
+ else
+ expand_omp_taskloop_for_outer (region, &fd, inner_stmt);
+ }
+ else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
+ && !fd.have_ordered)
+ {
+ if (fd.chunk_size == NULL)
+ expand_omp_for_static_nochunk (region, &fd, inner_stmt);
+ else
+ expand_omp_for_static_chunk (region, &fd, inner_stmt);
+ }
+ else
+ {
+ int fn_index, start_ix, next_ix;
+
+ gcc_assert (gimple_omp_for_kind (fd.for_stmt)
+ == GF_OMP_FOR_KIND_FOR);
+ if (fd.chunk_size == NULL
+ && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
+ fd.chunk_size = integer_zero_node;
+ gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
+ fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
+ ? 3 : fd.sched_kind;
+ if (!fd.ordered)
+ fn_index += fd.have_ordered * 4;
+ if (fd.ordered)
+ start_ix = ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START) + fn_index;
+ else
+ start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
+ next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
+ if (fd.iter_type == long_long_unsigned_type_node)
+ {
+ start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
+ - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
+ next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
+ - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
+ }
+ expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
+ (enum built_in_function) next_ix, inner_stmt);
+ }
+
+ if (gimple_in_ssa_p (cfun))
+ update_ssa (TODO_update_ssa_only_virtuals);
+}
+
+
+/* Expand code for an OpenMP sections directive. In pseudo code, we generate
-/* Expand code for an OpenMP sections directive. In pseudo code, we generate
-
v = GOMP_sections_start (n);
L0:
switch (v)
@@ -13375,6 +14252,7 @@ lower_omp_for (gimple_stmt_iterator *gsi
gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
gbind *new_stmt;
gimple_seq omp_for_body, body, dlist;
+ gimple_seq oacc_head = NULL, oacc_tail = NULL;
size_t i;
push_gimplify_context ();
@@ -13483,6 +14361,16 @@ lower_omp_for (gimple_stmt_iterator *gsi
/* Once lowered, extract the bounds and clauses. */
extract_omp_for_data (stmt, &fd, NULL);
+ if (is_gimple_omp_oacc (ctx->stmt)
+ && !ctx_in_oacc_kernels_region (ctx))
+ lower_oacc_head_tail (gimple_location (stmt),
+ gimple_omp_for_clauses (stmt),
+ &oacc_head, &oacc_tail, ctx);
+
+ /* Add OpenACC partitioning markers just before the loop */
+ if (oacc_head)
+ gimple_seq_add_seq (&body, oacc_head);
+
lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
@@ -13516,6 +14404,11 @@ lower_omp_for (gimple_stmt_iterator *gsi
/* Region exit marker goes at the end of the loop body. */
gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
maybe_add_implicit_barrier_cancel (ctx, &body);
+
+ /* Add OpenACC joining and reduction markers just after the loop. */
+ if (oacc_tail)
+ gimple_seq_add_seq (&body, oacc_tail);
+
pop_gimplify_context (new_stmt);
gimple_bind_append_vars (new_stmt, ctx->block_vars);
Index: gcc/internal-fn.def
===================================================================
--- gcc/internal-fn.def (revision 229101)
+++ gcc/internal-fn.def (working copy)
@@ -65,11 +65,56 @@ DEF_INTERNAL_FN (SUB_OVERFLOW, ECF_CONST
DEF_INTERNAL_FN (MUL_OVERFLOW, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (TSAN_FUNC_EXIT, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (VA_ARG, ECF_NOTHROW | ECF_LEAF, NULL)
/* An unduplicable, uncombinable function. Generally used to preserve
a CFG property in the face of jump threading, tail merging or
other such optimizations. The first argument distinguishes
between uses. Other arguments are as needed for use. The return
type depends on use too. */
DEF_INTERNAL_FN (UNIQUE, ECF_NOTHROW | ECF_LEAF, NULL)
#define IFN_UNIQUE_UNSPEC 0 /* Undifferentiated UNIQUE. */
+
+/* FORK and JOIN mark the points at which OpenACC partitioned
+ execution is entered or exited. They take an INTEGER_CST argument,
+ indicating the axis of forking or joining and return nothing. */
+#define IFN_UNIQUE_OACC_FORK 1
+#define IFN_UNIQUE_OACC_JOIN 2
+/* HEAD_MARK and TAIL_MARK are used to demark the sequence entering or
+ leaving partitioned execution. */
+#define IFN_UNIQUE_OACC_HEAD_MARK 3
+#define IFN_UNIQUE_OACC_TAIL_MARK 4
+
+/* DIM_SIZE and DIM_POS return the size of a particular compute
+ dimension and the executing thread's position within that
+ dimension. DIM_POS is pure (and not const) so that it isn't
+ thought to clobber memory and can be gcse'd within a single
+ parallel region, but not across FORK/JOIN boundaries. They take a
+ single INTEGER_CST argument. */
+DEF_INTERNAL_FN (GOACC_DIM_SIZE, ECF_CONST | ECF_NOTHROW | ECF_LEAF, ".")
+DEF_INTERNAL_FN (GOACC_DIM_POS, ECF_PURE | ECF_NOTHROW | ECF_LEAF, ".")
+
+/* OpenACC looping abstraction. Allows the precise stepping of
+ the compute geometry over the loop iterations to be deferred until
+ it is known which compiler is generating the code. The action is
+ encoded in a constant first argument.
+
+ CHUNK_MAX = LOOP (CODE_CHUNKS, DIR, RANGE, STEP, CHUNK_SIZE, MASK)
+ STEP = LOOP (CODE_STEP, DIR, RANGE, STEP, CHUNK_SIZE, MASK)
+ OFFSET = LOOP (CODE_OFFSET, DIR, RANGE, STEP, CHUNK_SIZE, MASK, CHUNK_NO)
+ BOUND = LOOP (CODE_BOUND, DIR, RANGE, STEP, CHUNK_SIZE, MASK, OFFSET)
+
+ DIR - +1 for up loop, -1 for down loop
+ RANGE - Range of loop (END - BASE)
+ STEP - iteration step size
+ CHUNKING - size of chunking, (constant zero for no chunking)
+ CHUNK_NO - chunk number
+ MASK - partitioning mask.
+
+ TODO: The partitioning mask and chunk size are a transition stage,
+ they will be removed once the required infrastructure is in place. */
+
+DEF_INTERNAL_FN (GOACC_LOOP, ECF_PURE | ECF_NOTHROW, NULL)
+#define IFN_GOACC_LOOP_CHUNKS 0 /* Number of chunks. */
+#define IFN_GOACC_LOOP_STEP 1 /* Size of each thread's step. */
+#define IFN_GOACC_LOOP_OFFSET 2 /* Initial iteration value. */
+#define IFN_GOACC_LOOP_BOUND 3 /* Limit of iteration value. */
Index: gcc/internal-fn.c
===================================================================
--- gcc/internal-fn.c (revision 229101)
+++ gcc/internal-fn.c (working copy)
@@ -1958,28 +1958,95 @@ expand_VA_ARG (gcall *stmt ATTRIBUTE_UNU
gcc_unreachable ();
}
/* Expand the IFN_UNIQUE function according to its first argument. */
static void
expand_UNIQUE (gcall *stmt)
{
rtx pattern = NULL_RTX;
switch (TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)))
{
default:
gcc_unreachable ();
case IFN_UNIQUE_UNSPEC:
#ifdef HAVE_unique
pattern = gen_unique ();
#endif
break;
+
+ case IFN_UNIQUE_OACC_FORK:
+#ifdef HAVE_oacc_fork
+ pattern = expand_normal (gimple_call_arg (stmt, 1));
+ pattern = gen_oacc_fork (pattern);
+#else
+ gcc_unreachable ();
+#endif
+ break;
+
+ case IFN_UNIQUE_OACC_JOIN:
+#ifdef HAVE_oacc_join
+ pattern = expand_normal (gimple_call_arg (stmt, 1));
+ pattern = gen_oacc_join (pattern);
+#else
+ gcc_unreachable ();
+#endif
+ break;
}
if (pattern)
emit_insn (pattern);
}
+
+/* The size of an OpenACC compute dimension. */
+
+static void
+expand_GOACC_DIM_SIZE (gcall *ARG_UNUSED (stmt))
+{
+ tree lhs = gimple_call_lhs (stmt);
+
+ if (!lhs)
+ return;
+
+ rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+#ifdef HAVE_oacc_dim_size
+ rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
+ VOIDmode, EXPAND_NORMAL);
+ emit_insn (gen_oacc_dim_size (target, dim));
+#else
+ emit_move_insn (target, GEN_INT (1));
+#endif
+}
+
+/* The position of an OpenACC execution engine along one compute axis. */
+
+static void
+expand_GOACC_DIM_POS (gcall *ARG_UNUSED (stmt))
+{
+ tree lhs = gimple_call_lhs (stmt);
+
+ if (!lhs)
+ return;
+
+ rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+#ifdef HAVE_oacc_dim_pos
+ rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
+ VOIDmode, EXPAND_NORMAL);
+ emit_insn (gen_oacc_dim_pos (target, dim));
+#else
+ emit_move_insn (target, const0_rtx);
+#endif
+}
+
+/* This is expanded by oacc_device_lower pass. */
+
+static void
+expand_GOACC_LOOP (gcall *stmt ATTRIBUTE_UNUSED)
+{
+ gcc_unreachable ();
+}
+
/* Routines to expand each internal function, indexed by function number.
Each routine has the prototype: