[PATCH 4/n] OpenMP 4.0 offloading infrastructure: lto-wrapper
Ilya Verbin
iverbin@gmail.com
Thu Oct 2 15:15:00 GMT 2014
Hello,
With this patch lto-wrapper performs invocation of mkoffload tool for each
offload target. This tool should be provided by the corresponding offload
compiler. It will compile IR from .gnu.offload_lto_* sections into offload
target code and embed the resultant code (offload image) into the new host's
object file.
Bootstrapped and regtested on top of patch 3. Is it OK for trunk?
Thanks,
-- Ilya
2014-10-02 Ilya Verbin <ilya.verbin@intel.com>
Bernd Schmidt <bernds@codesourcery.com>
Andrey Turetskiy <andrey.turetskiy@intel.com>
Michael Zolotukhin <michael.v.zolotukhin@intel.com>
gcc/
* gcc.c (spec_host_machine, accel_dir_suffix): New variables.
(process_command): Tweak path construction for the possibility
of being configured as an offload compiler.
(main): Likewise. Look up specs in just_machine_suffix only if not
ACCEL_COMPILER. Construct OFFLOAD_TARGET_NAMES environment variable if
we have OFFLOAD_TARGETS.
* lto-wrapper.c (OFFLOAD_TARGET_NAMES_ENV): Define.
(offload_names, offloadbegin, offloadend): New static variables.
(free_array_of_ptrs, parse_env_var, access_check, compile_offload_image)
(compile_images_for_offload_targets, copy_file, find_offloadbeginend):
New static functions.
(run_gcc): Determine whether offload sections are present. If so, run
compile_images_for_offload_targets and return the names of new generated
objects to linker. If there are offload sections, but no LTO sections,
then return the copies of input objects without link-time recompilation.
lto-plugin/
* lto-plugin.c (OFFLOAD_SECTION, OFFLOAD_SECTION_LEN): Define.
(struct plugin_objfile): Add new field "offload".
(process_offload_section): New static function.
(claim_file_handler): Claim file if it contains offload sections.
---
diff --git a/gcc/gcc.c b/gcc/gcc.c
index 47c4e28..82509a8 100644
--- a/gcc/gcc.c
+++ b/gcc/gcc.c
@@ -157,6 +157,7 @@ static const char *const spec_version = DEFAULT_TARGET_VERSION;
/* The target machine. */
static const char *spec_machine = DEFAULT_TARGET_MACHINE;
+static const char *spec_host_machine = DEFAULT_REAL_TARGET_MACHINE;
/* Nonzero if cross-compiling.
When -b is used, the value comes from the `specs' file. */
@@ -1296,6 +1297,9 @@ static const char *const standard_startfile_prefix_2
relative to the driver. */
static const char *const tooldir_base_prefix = TOOLDIR_BASE_PREFIX;
+/* A prefix to be used when this is an accelerator compiler. */
+static const char *const accel_dir_suffix = ACCEL_DIR_SUFFIX;
+
/* Subdirectory to use for locating libraries. Set by
set_multilib_dir based on the compilation options. */
@@ -4122,15 +4126,15 @@ process_command (unsigned int decoded_options_count,
}
gcc_assert (!IS_ABSOLUTE_PATH (tooldir_base_prefix));
- tooldir_prefix2 = concat (tooldir_base_prefix, spec_machine,
+ tooldir_prefix2 = concat (tooldir_base_prefix, spec_host_machine,
dir_separator_str, NULL);
/* Look for tools relative to the location from which the driver is
running, or, if that is not available, the configured prefix. */
tooldir_prefix
= concat (gcc_exec_prefix ? gcc_exec_prefix : standard_exec_prefix,
- spec_machine, dir_separator_str,
- spec_version, dir_separator_str, tooldir_prefix2, NULL);
+ spec_host_machine, dir_separator_str, spec_version,
+ accel_dir_suffix, dir_separator_str, tooldir_prefix2, NULL);
free (tooldir_prefix2);
add_prefix (&exec_prefixes,
@@ -6878,8 +6882,8 @@ main (int argc, char **argv)
/* Read specs from a file if there is one. */
- machine_suffix = concat (spec_machine, dir_separator_str,
- spec_version, dir_separator_str, NULL);
+ machine_suffix = concat (spec_host_machine, dir_separator_str, spec_version,
+ accel_dir_suffix, dir_separator_str, NULL);
just_machine_suffix = concat (spec_machine, dir_separator_str, NULL);
specs_file = find_a_file (&startfile_prefixes, "specs", R_OK, true);
@@ -6889,16 +6893,17 @@ main (int argc, char **argv)
else
init_spec ();
+#ifndef ACCEL_COMPILER
/* We need to check standard_exec_prefix/just_machine_suffix/specs
for any override of as, ld and libraries. */
specs_file = (char *) alloca (strlen (standard_exec_prefix)
+ strlen (just_machine_suffix) + sizeof ("specs"));
-
strcpy (specs_file, standard_exec_prefix);
strcat (specs_file, just_machine_suffix);
strcat (specs_file, "specs");
if (access (specs_file, R_OK) == 0)
read_specs (specs_file, true, false);
+#endif
/* Process any configure-time defaults specified for the command line
options, via OPTION_DEFAULT_SPECS. */
@@ -7077,8 +7082,9 @@ main (int argc, char **argv)
/* If we have a GCC_EXEC_PREFIX envvar, modify it for cpp's sake. */
if (gcc_exec_prefix)
- gcc_exec_prefix = concat (gcc_exec_prefix, spec_machine, dir_separator_str,
- spec_version, dir_separator_str, NULL);
+ gcc_exec_prefix = concat (gcc_exec_prefix, spec_host_machine,
+ dir_separator_str, spec_version,
+ accel_dir_suffix, dir_separator_str, NULL);
/* Now we have the specs.
Set the `valid' bits for switches that match anything in any spec. */
@@ -7097,6 +7103,16 @@ main (int argc, char **argv)
obstack_grow (&collect_obstack, argv[0], strlen (argv[0]) + 1);
xputenv (XOBFINISH (&collect_obstack, char *));
+ if (strlen (OFFLOAD_TARGETS) > 0)
+ {
+ obstack_init (&collect_obstack);
+ obstack_grow (&collect_obstack, "OFFLOAD_TARGET_NAMES=",
+ sizeof ("OFFLOAD_TARGET_NAMES=") - 1);
+ obstack_grow (&collect_obstack, OFFLOAD_TARGETS,
+ strlen (OFFLOAD_TARGETS) + 1);
+ xputenv (XOBFINISH (&collect_obstack, char *));
+ }
+
/* Set up to remember the pathname of the lto wrapper. */
if (have_c)
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 08fd090..2c9b503 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -49,6 +49,8 @@ along with GCC; see the file COPYING3. If not see
#include "lto-section-names.h"
#include "collect-utils.h"
+#define OFFLOAD_TARGET_NAMES_ENV "OFFLOAD_TARGET_NAMES"
+
enum lto_mode_d {
LTO_MODE_NONE, /* Not doing LTO. */
LTO_MODE_LTO, /* Normal LTO. */
@@ -63,6 +65,8 @@ static char *flto_out;
static unsigned int nr;
static char **input_names;
static char **output_names;
+static char **offload_names;
+static const char *offloadbegin, *offloadend;
static char *makefile;
const char tool_name[] = "lto-wrapper";
@@ -358,6 +362,224 @@ merge_and_complain (struct cl_decoded_option **decoded_options,
}
}
+/* Auxiliary function that frees elements of PTR and PTR itself.
+ N is number of elements to be freed. If PTR is NULL, nothing is freed.
+ If an element is NULL, subsequent elements are not freed. */
+
+static void **
+free_array_of_ptrs (void **ptr, unsigned n)
+{
+ if (!ptr)
+ return NULL;
+ for (unsigned i = 0; i < n; i++)
+ {
+ if (!ptr[i])
+ break;
+ free (ptr[i]);
+ }
+ free (ptr);
+ return NULL;
+}
+
+/* Parse STR, saving found tokens into PVALUES and return their number.
+ Tokens are assumed to be delimited by ':'. If APPEND is non-null,
+ append it to every token we find. */
+
+static unsigned
+parse_env_var (const char *str, char ***pvalues, const char *append)
+{
+ const char *curval, *nextval;
+ char **values;
+ unsigned num = 1, i;
+
+ curval = strchr (str, ':');
+ while (curval)
+ {
+ num++;
+ curval = strchr (curval + 1, ':');
+ }
+
+ values = (char**) xmalloc (num * sizeof (char*));
+ curval = str;
+ nextval = strchrnul (curval, ':');
+
+ int append_len = append ? strlen (append) : 0;
+ for (i = 0; i < num; i++)
+ {
+ int l = nextval - curval;
+ values[i] = (char*) xmalloc (l + 1 + append_len);
+ memcpy (values[i], curval, l);
+ values[i][l] = 0;
+ if (append)
+ strcat (values[i], append);
+ curval = nextval + 1;
+ nextval = strchrnul (curval, ':');
+ }
+ *pvalues = values;
+ return num;
+}
+
+/* Check whether NAME can be accessed in MODE. This is like access,
+ except that it never considers directories to be executable. */
+
+static int
+access_check (const char *name, int mode)
+{
+ if (mode == X_OK)
+ {
+ struct stat st;
+
+ if (stat (name, &st) < 0
+ || S_ISDIR (st.st_mode))
+ return -1;
+ }
+
+ return access (name, mode);
+}
+
+/* Prepare a target image for offload TARGET, using mkoffload tool from
+ COMPILER_PATH. Return the name of the resultant object file. */
+
+static char *
+compile_offload_image (const char *target, const char *compiler_path,
+ unsigned in_argc, char *in_argv[])
+{
+ char *filename = NULL;
+ char **argv;
+ char *suffix
+ = XALLOCAVEC (char, strlen ("/accel//mkoffload") + strlen (target) + 1);
+ strcpy (suffix, "/accel/");
+ strcat (suffix, target);
+ strcat (suffix, "/mkoffload");
+
+ char **paths = NULL;
+ unsigned n_paths = parse_env_var (compiler_path, &paths, suffix);
+
+ const char *compiler = NULL;
+ for (unsigned i = 0; i < n_paths; i++)
+ if (access_check (paths[i], X_OK) == 0)
+ {
+ compiler = paths[i];
+ break;
+ }
+
+ if (!compiler)
+ goto out;
+
+ /* Generate temporary output file name. */
+ filename = make_temp_file (".target.o");
+
+ struct obstack argv_obstack;
+ obstack_init (&argv_obstack);
+ obstack_ptr_grow (&argv_obstack, compiler);
+ obstack_ptr_grow (&argv_obstack, "-o");
+ obstack_ptr_grow (&argv_obstack, filename);
+
+ for (unsigned i = 1; i < in_argc; i++)
+ obstack_ptr_grow (&argv_obstack, in_argv[i]);
+ obstack_ptr_grow (&argv_obstack, NULL);
+
+ argv = XOBFINISH (&argv_obstack, char **);
+ fork_execute (argv[0], argv, true);
+ obstack_free (&argv_obstack, NULL);
+
+ out:
+ free_array_of_ptrs ((void **) paths, n_paths);
+ return filename;
+}
+
+
+/* The main routine dealing with offloading.
+ The routine builds a target image for each offload target. IN_ARGC and
+ IN_ARGV specify options and input object files. As all of them could contain
+ target sections, we pass them all to target compilers. */
+
+static void
+compile_images_for_offload_targets (unsigned in_argc, char *in_argv[])
+{
+ char **names = NULL;
+ const char *target_names = getenv (OFFLOAD_TARGET_NAMES_ENV);
+ if (!target_names)
+ return;
+ unsigned num_targets = parse_env_var (target_names, &names, NULL);
+
+ const char *compiler_path = getenv ("COMPILER_PATH");
+ if (!compiler_path)
+ goto out;
+
+ /* Prepare an image for each target and save the name of the resultant object
+ file to the OFFLOAD_NAMES array. It is terminated by a NULL entry. */
+ offload_names = XCNEWVEC (char *, num_targets + 1);
+ for (unsigned i = 0; i < num_targets; i++)
+ {
+ offload_names[i] = compile_offload_image (names[i], compiler_path,
+ in_argc, in_argv);
+ if (!offload_names[i])
+ fatal_error ("problem with building target image for %s\n", names[i]);
+ }
+
+ out:
+ free_array_of_ptrs ((void **) names, num_targets);
+}
+
+/* Copy a file from SRC to DEST. */
+
+static void
+copy_file (const char *dest, const char *src)
+{
+ FILE *d = fopen (dest, "wb");
+ FILE *s = fopen (src, "rb");
+ char buffer[512];
+ while (!feof (s))
+ {
+ size_t len = fread (buffer, 1, 512, s);
+ if (ferror (s) != 0)
+ fatal_error ("reading input file");
+ if (len > 0)
+ {
+ fwrite (buffer, 1, len, d);
+ if (ferror (d) != 0)
+ fatal_error ("writing output file");
+ }
+ }
+}
+
+/* Find the crtoffloadbegin.o and crtoffloadend.o files in LIBRARY_PATH, make
+ copies and store the names of the copies in offloadbegin and offloadend. */
+
+static void
+find_offloadbeginend (void)
+{
+ char **paths = NULL;
+ const char *library_path = getenv ("LIBRARY_PATH");
+ if (!library_path)
+ return;
+ unsigned n_paths = parse_env_var (library_path, &paths, "/crtoffloadbegin.o");
+
+ unsigned i;
+ for (i = 0; i < n_paths; i++)
+ if (access_check (paths[i], R_OK) == 0)
+ {
+ size_t len = strlen (paths[i]);
+ char *tmp = xstrdup (paths[i]);
+ strcpy (paths[i] + len - strlen ("begin.o"), "end.o");
+ if (access_check (paths[i], R_OK) != 0)
+ fatal_error ("installation error, can't find crtoffloadend.o");
+ /* The linker will delete the filenames we give it, so make
+ copies. */
+ offloadbegin = make_temp_file (".o");
+ offloadend = make_temp_file (".o");
+ copy_file (offloadbegin, tmp);
+ copy_file (offloadend, paths[i]);
+ free (tmp);
+ break;
+ }
+ if (i == n_paths)
+ fatal_error ("installation error, can't find crtoffloadbegin.o");
+
+ free_array_of_ptrs ((void **) paths, n_paths);
+}
+
/* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
static void
@@ -378,6 +600,8 @@ run_gcc (unsigned argc, char *argv[])
unsigned int decoded_options_count;
struct obstack argv_obstack;
int new_head_argc;
+ bool have_lto = false;
+ bool have_offload = false;
/* Get the driver and options. */
collect_gcc = getenv ("COLLECT_GCC");
@@ -426,6 +650,9 @@ run_gcc (unsigned argc, char *argv[])
close (fd);
continue;
}
+ if (simple_object_find_section (sobj, OFFLOAD_SECTION_NAME_PREFIX ".opts",
+ &offset, &length, &errmsg, &err))
+ have_offload = true;
if (!simple_object_find_section (sobj, LTO_SECTION_NAME_PREFIX "." "opts",
&offset, &length, &errmsg, &err))
{
@@ -433,6 +660,7 @@ run_gcc (unsigned argc, char *argv[])
close (fd);
continue;
}
+ have_lto = true;
lseek (fd, file_offset + offset, SEEK_SET);
data = (char *)xmalloc (length);
read (fd, data, length);
@@ -624,6 +852,54 @@ run_gcc (unsigned argc, char *argv[])
/* Remember at which point we can scrub args to re-use the commons. */
new_head_argc = obstack_object_size (&argv_obstack) / sizeof (void *);
+ if (have_offload)
+ {
+ compile_images_for_offload_targets (argc, argv);
+ if (offload_names)
+ {
+ find_offloadbeginend ();
+ for (i = 0; offload_names[i]; i++)
+ printf ("%s\n", offload_names[i]);
+ free_array_of_ptrs ((void **) offload_names, i);
+ }
+ }
+
+ if (offloadbegin)
+ printf ("%s\n", offloadbegin);
+
+ /* If object files contain offload sections, but do not contain LTO sections,
+ then there is no need to perform a link-time recompilation, i.e.
+ lto-wrapper is used only for a compilation of offload images. */
+ if (have_offload && !have_lto)
+ {
+ for (i = 1; i < argc; ++i)
+ if (strncmp (argv[i], "-fresolution=", sizeof ("-fresolution=") - 1))
+ {
+ char *out_file;
+ /* Can be ".o" or ".so". */
+ char *ext = strrchr (argv[i], '.');
+ if (ext == NULL)
+ out_file = make_temp_file ("");
+ else
+ out_file = make_temp_file (ext);
+ /* The linker will delete the files we give it, so make copies. */
+ copy_file (out_file, argv[i]);
+ printf ("%s\n", out_file);
+ }
+
+ /* By default linker does not discard .gnu.offload_lto_* sections. */
+ const char *linker_script = make_temp_file ("_linker_script.x");
+ FILE *stream = fopen (linker_script, "w");
+ if (!stream)
+ fatal_error ("fopen %s: %m", linker_script);
+ fprintf (stream, "SECTIONS { /DISCARD/ : { *("
+ OFFLOAD_SECTION_NAME_PREFIX "*) } }\n");
+ fclose (stream);
+ printf ("%s\n", linker_script);
+
+ goto finish;
+ }
+
if (lto_mode == LTO_MODE_LTO)
{
flto_out = make_temp_file (".lto.o");
@@ -850,6 +1126,10 @@ cont:
obstack_free (&env_obstack, NULL);
}
+ finish:
+ if (offloadend)
+ printf ("%s\n", offloadend);
+
obstack_free (&argv_obstack, NULL);
}
diff --git a/lto-plugin/lto-plugin.c b/lto-plugin/lto-plugin.c
index 910e23c..fb6555d 100644
--- a/lto-plugin/lto-plugin.c
+++ b/lto-plugin/lto-plugin.c
@@ -86,6 +86,8 @@ along with this program; see the file COPYING3. If not see
#define LTO_SECTION_PREFIX ".gnu.lto_.symtab"
#define LTO_SECTION_PREFIX_LEN (sizeof (LTO_SECTION_PREFIX) - 1)
+#define OFFLOAD_SECTION ".gnu.offload_lto_.opts"
+#define OFFLOAD_SECTION_LEN (sizeof (OFFLOAD_SECTION) - 1)
/* The part of the symbol table the plugin has to keep track of. Note that we
must keep SYMS until all_symbols_read is called to give the linker time to
@@ -111,6 +113,7 @@ struct plugin_symtab
struct plugin_objfile
{
int found;
+ int offload;
simple_object_read *objfile;
struct plugin_symtab *out;
const struct ld_plugin_input_file *file;
@@ -862,6 +865,21 @@ err:
return 0;
}
+/* Find an offload section of an object file. */
+
+static int
+process_offload_section (void *data, const char *name, off_t offset, off_t len)
+{
+ if (!strncmp (name, OFFLOAD_SECTION, OFFLOAD_SECTION_LEN))
+ {
+ struct plugin_objfile *obj = (struct plugin_objfile *) data;
+ obj->offload = 1;
+ return 0;
+ }
+
+ return 1;
+}
+
/* Callback used by gold to check if the plugin will claim FILE. Writes
the result in CLAIMED. */
@@ -899,6 +917,7 @@ claim_file_handler (const struct ld_plugin_input_file *file, int *claimed)
*claimed = 0;
obj.file = file;
obj.found = 0;
+ obj.offload = 0;
obj.out = <o_file.symtab;
errmsg = NULL;
obj.objfile = simple_object_start_read (file->fd, file->offset, LTO_SEGMENT_NAME,
@@ -920,7 +939,11 @@ claim_file_handler (const struct ld_plugin_input_file *file, int *claimed)
goto err;
}
- if (obj.found == 0)
+ if (obj.objfile)
+ simple_object_find_sections (obj.objfile, process_offload_section,
+ &obj, &err);
+
+ if (obj.found == 0 && obj.offload == 0)
goto err;
if (obj.found > 1)
--
1.7.1
More information about the Gcc-patches
mailing list