Re: [PATCH] RFC: machine-readable diagnostic output (PR other/19165)

[Richard Biener <richard dot guenther at gmail dot com>]

On Tue, Nov 13, 2018 at 8:58 AM David Malcolm <dmalcolm@redhat.com> wrote:
>
> This patch implements a -fdiagnostics-format=json option which
> converts the diagnostics to be output to stderr in a JSON format;
> see the documentation in invoke.texi.
>
> Logically-related diagnostics are nested at the JSON level, using
> the auto_diagnostic_group mechanism.

LGTM if people really want it.

Richard.

> gcc/ChangeLog:
>         PR other/19165
>         * Makefile.in (OBJS): Move json.o to...
>         (OBJS-libcommon): ...here and add diagnostic-format-json.o.
>         * common.opt (fdiagnostics-format=): New option.
>         (diagnostics_output_format): New enum.
>         * diagnostic-format-json.cc: New file.
>         * diagnostic.c (default_diagnostic_final_cb): New function, taken
>         from start of diagnostic_finish.
>         (diagnostic_initialize): Initialize final_cb to
>         default_diagnostic_final_cb.
>         (diagnostic_finish): Move "being treated as errors" messages to
>         default_diagnostic_final_cb.  Call any final_cb.
>         * diagnostic.h (enum diagnostics_output_format): New enum.
>         (struct diagnostic_context): Add "final_cb".
>         (diagnostic_output_format_init): New decl.
>         * doc/invoke.texi (-fdiagnostics-format): New option.
>         * dwarf2out.c (gen_producer_string): Ignore
>         OPT_fdiagnostics_format_.
>         * gcc.c (driver_handle_option): Handle OPT_fdiagnostics_format_.
>         * lto-wrapper.c (append_diag_options): Ignore it.
>         * opts.c (common_handle_option): Handle it.
> ---
>  gcc/Makefile.in               |   2 +-
>  gcc/common.opt                |  17 +++
>  gcc/diagnostic-format-json.cc | 265 ++++++++++++++++++++++++++++++++++++++++++
>  gcc/diagnostic.c              |  40 ++++---
>  gcc/diagnostic.h              |  16 +++
>  gcc/doc/invoke.texi           |  78 +++++++++++++
>  gcc/dwarf2out.c               |   1 +
>  gcc/gcc.c                     |   5 +
>  gcc/lto-wrapper.c             |   1 +
>  gcc/opts.c                    |   5 +
>  10 files changed, 414 insertions(+), 16 deletions(-)
>  create mode 100644 gcc/diagnostic-format-json.cc
>
> diff --git a/gcc/Makefile.in b/gcc/Makefile.in
> index 16c9ed6..9534d59 100644
> --- a/gcc/Makefile.in
> +++ b/gcc/Makefile.in
> @@ -1395,7 +1395,6 @@ OBJS = \
>         ira-color.o \
>         ira-emit.o \
>         ira-lives.o \
> -       json.o \
>         jump.o \
>         langhooks.o \
>         lcm.o \
> @@ -1619,6 +1618,7 @@ OBJS = \
>  # Objects in libcommon.a, potentially used by all host binaries and with
>  # no target dependencies.
>  OBJS-libcommon = diagnostic.o diagnostic-color.o diagnostic-show-locus.o \
> +       diagnostic-format-json.o json.o \
>         edit-context.o \
>         pretty-print.o intl.o \
>         sbitmap.o \
> diff --git a/gcc/common.opt b/gcc/common.opt
> index 5a5d332..2f669f6 100644
> --- a/gcc/common.opt
> +++ b/gcc/common.opt
> @@ -1273,6 +1273,23 @@ Enum(diagnostic_color_rule) String(always) Value(DIAGNOSTICS_COLOR_YES)
>  EnumValue
>  Enum(diagnostic_color_rule) String(auto) Value(DIAGNOSTICS_COLOR_AUTO)
>
> +fdiagnostics-format=
> +Common Joined RejectNegative Enum(diagnostics_output_format)
> +-fdiagnostics-format=[text|json] Select output format
> +
> +; Required for these enum values.
> +SourceInclude
> +diagnostic.h
> +
> +Enum
> +Name(diagnostics_output_format) Type(int)
> +
> +EnumValue
> +Enum(diagnostics_output_format) String(text) Value(DIAGNOSTICS_OUTPUT_FORMAT_TEXT)
> +
> +EnumValue
> +Enum(diagnostics_output_format) String(json) Value(DIAGNOSTICS_OUTPUT_FORMAT_JSON)
> +
>  fdiagnostics-parseable-fixits
>  Common Var(flag_diagnostics_parseable_fixits)
>  Print fix-it hints in machine-readable form.
> diff --git a/gcc/diagnostic-format-json.cc b/gcc/diagnostic-format-json.cc
> new file mode 100644
> index 0000000..7860696
> --- /dev/null
> +++ b/gcc/diagnostic-format-json.cc
> @@ -0,0 +1,265 @@
> +/* JSON output for diagnostics
> +   Copyright (C) 2018 Free Software Foundation, Inc.
> +   Contributed by David Malcolm <dmalcolm@redhat.com>.
> +
> +This file is part of GCC.
> +
> +GCC is free software; you can redistribute it and/or modify it under
> +the terms of the GNU General Public License as published by the Free
> +Software Foundation; either version 3, or (at your option) any later
> +version.
> +
> +GCC is distributed in the hope that it will be useful, but WITHOUT ANY
> +WARRANTY; without even the implied warranty of MERCHANTABILITY or
> +FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
> +for more details.
> +
> +You should have received a copy of the GNU General Public License
> +along with GCC; see the file COPYING3.  If not see
> +<http://www.gnu.org/licenses/>.  */
> +
> +
> +#include "config.h"
> +#include "system.h"
> +#include "coretypes.h"
> +#include "diagnostic.h"
> +#include "json.h"
> +
> +/* The top-level JSON array of pending diagnostics.  */
> +
> +static json::array *toplevel_array;
> +
> +/* The JSON object for the current diagnostic group.  */
> +
> +static json::object *cur_group;
> +
> +/* The JSON array for the "children" array within the current diagnostic
> +   group.  */
> +
> +static json::array *cur_children_array;
> +
> +/* Generate a JSON object for LOC.  */
> +
> +static json::object *
> +json_from_expanded_location (location_t loc)
> +{
> +  expanded_location exploc = expand_location (loc);
> +  json::object *result = new json::object ();
> +  result->set ("file", new json::string (exploc.file));
> +  result->set ("line", new json::number (exploc.line));
> +  result->set ("column", new json::number (exploc.column));
> +  return result;
> +}
> +
> +/* Generate a JSON object for LOC_RANGE.  */
> +
> +static json::object *
> +json_from_location_range (const location_range *loc_range, unsigned range_idx)
> +{
> +  location_t caret_loc = get_pure_location (loc_range->m_loc);
> +
> +  if (caret_loc == UNKNOWN_LOCATION)
> +    return NULL;
> +
> +  location_t start_loc = get_start (loc_range->m_loc);
> +  location_t finish_loc = get_finish (loc_range->m_loc);
> +
> +  json::object *result = new json::object ();
> +  result->set ("caret", json_from_expanded_location (caret_loc));
> +  if (start_loc != caret_loc)
> +    result->set ("start", json_from_expanded_location (start_loc));
> +  if (finish_loc != caret_loc)
> +    result->set ("finish", json_from_expanded_location (finish_loc));
> +
> +  if (loc_range->m_label)
> +    {
> +      label_text text;
> +      text = loc_range->m_label->get_text (range_idx);
> +      if (text.m_buffer)
> +       result->set ("label", new json::string (text.m_buffer));
> +      text.maybe_free ();
> +    }
> +
> +  return result;
> +}
> +
> +/* Generate a JSON object for HINT.  */
> +
> +static json::object *
> +json_from_fixit_hint (const fixit_hint *hint)
> +{
> +  json::object *fixit_obj = new json::object ();
> +
> +  location_t start_loc = hint->get_start_loc ();
> +  fixit_obj->set ("start", json_from_expanded_location (start_loc));
> +  location_t next_loc = hint->get_next_loc ();
> +  fixit_obj->set ("next", json_from_expanded_location (next_loc));
> +  fixit_obj->set ("string", new json::string (hint->get_string ()));
> +
> +  return fixit_obj;
> +}
> +
> +/* No-op implementation of "begin_diagnostic" for JSON output.  */
> +
> +static void
> +json_begin_diagnostic (diagnostic_context *, diagnostic_info *)
> +{
> +}
> +
> +/* Implementation of "end_diagnostic" for JSON output.
> +   Generate a JSON object for DIAGNOSTIC, and store for output
> +   within current diagnostic group.  */
> +
> +static void
> +json_end_diagnostic (diagnostic_context *context, diagnostic_info *diagnostic)
> +{
> +  json::object *diag_obj = new json::object ();
> +
> +  /* Get "kind" of diagnostic.  */
> +  {
> +    static const char *const diagnostic_kind_text[] = {
> +#define DEFINE_DIAGNOSTIC_KIND(K, T, C) (T),
> +#include "diagnostic.def"
> +#undef DEFINE_DIAGNOSTIC_KIND
> +      "must-not-happen"
> +    };
> +    /* Lose the trailing ": ".  */
> +    const char *kind_text = diagnostic_kind_text[diagnostic->kind];
> +    size_t len = strlen (kind_text);
> +    gcc_assert (len > 2);
> +    gcc_assert (kind_text[len - 2] == ':');
> +    gcc_assert (kind_text[len - 1] == ' ');
> +    char *rstrip = xstrdup (kind_text);
> +    rstrip[len - 2] = '\0';
> +    diag_obj->set ("kind", new json::string (rstrip));
> +    free (rstrip);
> +  }
> +
> +  // FIXME: encoding of the message (json::string requires UTF-8)
> +  diag_obj->set ("message",
> +                new json::string (pp_formatted_text (context->printer)));
> +  pp_clear_output_area (context->printer);
> +
> +  // FIXME: get this from inside diagnostic_report_diagnostic
> +  diagnostic_t orig_diag_kind = diagnostic->kind;
> +  char *option_text;
> +  option_text = context->option_name (context, diagnostic->option_index,
> +                                     orig_diag_kind, diagnostic->kind);
> +  if (option_text)
> +    {
> +      diag_obj->set ("option", new json::string (option_text));
> +      free (option_text);
> +    }
> +
> +  /* If we've already emitted a diagnostic within this auto_diagnostic_group,
> +     then add diag_obj to its "children" array.  */
> +  if (cur_group)
> +    {
> +      gcc_assert (cur_children_array);
> +      cur_children_array->append (diag_obj);
> +    }
> +  else
> +    {
> +      /* Otherwise, make diag_obj be the top-level object within the group;
> +        add a "children" array.  */
> +      toplevel_array->append (diag_obj);
> +      cur_group = diag_obj;
> +      cur_children_array = new json::array ();
> +      diag_obj->set ("children", cur_children_array);
> +    }
> +
> +  const rich_location *richloc = diagnostic->richloc;
> +
> +  json::array *loc_array = new json::array ();
> +  diag_obj->set ("locations", loc_array);
> +
> +  for (unsigned int i = 0; i < richloc->get_num_locations (); i++)
> +    {
> +      const location_range *loc_range = richloc->get_range (i);
> +      json::object *loc_obj = json_from_location_range (loc_range, i);
> +      if (loc_obj)
> +       loc_array->append (loc_obj);
> +    }
> +
> +  if (richloc->get_num_fixit_hints ())
> +    {
> +      json::array *fixit_array = new json::array ();
> +      diag_obj->set ("fixits", fixit_array);
> +      for (unsigned int i = 0; i < richloc->get_num_fixit_hints (); i++)
> +       {
> +         const fixit_hint *hint = richloc->get_fixit_hint (i);
> +         json::object *fixit_obj = json_from_fixit_hint (hint);
> +         fixit_array->append (fixit_obj);
> +       }
> +    }
> +
> +  /* TODO: tree-ish things:
> +     TODO: functions
> +     TODO: inlining information
> +     TODO: macro expansion information.  */
> +}
> +
> +/* No-op implementation of "begin_group_cb" for JSON output.  */
> +
> +static void
> +json_begin_group (diagnostic_context *)
> +{
> +}
> +
> +/* Implementation of "end_group_cb" for JSON output.  */
> +
> +static void
> +json_end_group (diagnostic_context *)
> +{
> +  cur_group = NULL;
> +  cur_children_array = NULL;
> +}
> +
> +/* Callback for final cleanup for JSON output.  */
> +
> +static void
> +json_final_cb (diagnostic_context *)
> +{
> +  /* Flush the top-level array.  */
> +  toplevel_array->dump (stderr);
> +  fprintf (stderr, "\n");
> +  delete toplevel_array;
> +  toplevel_array = NULL;
> +}
> +
> +/* Set the output format for CONTEXT to FORMAT.  */
> +
> +void
> +diagnostic_output_format_init (diagnostic_context *context,
> +                              enum diagnostics_output_format format)
> +{
> +  switch (format)
> +    {
> +    default:
> +      gcc_unreachable ();
> +    case DIAGNOSTICS_OUTPUT_FORMAT_TEXT:
> +      /* The default; do nothing.  */
> +      break;
> +
> +    case DIAGNOSTICS_OUTPUT_FORMAT_JSON:
> +      {
> +       /* Set up top-level JSON array.  */
> +       if (toplevel_array == NULL)
> +         toplevel_array = new json::array ();
> +
> +       /* Override callbacks.  */
> +       context->begin_diagnostic = json_begin_diagnostic;
> +       context->end_diagnostic = json_end_diagnostic;
> +       context->begin_group_cb = json_begin_group;
> +       context->end_group_cb =  json_end_group;
> +       context->final_cb =  json_final_cb;
> +
> +       /* The option is handled in JSON format, rather than as text.  */
> +       context->show_option_requested = false;
> +
> +       /* Don't colorize the text.  */
> +       pp_show_color (context->printer) = false;
> +      }
> +      break;
> +    }
> +}
> diff --git a/gcc/diagnostic.c b/gcc/diagnostic.c
> index 3d86a21..96d170d 100644
> --- a/gcc/diagnostic.c
> +++ b/gcc/diagnostic.c
> @@ -131,6 +131,28 @@ diagnostic_set_caret_max_width (diagnostic_context *context, int value)
>    context->caret_max_width = value;
>  }
>
> +/* Default implementation of final_cb.  */
> +
> +static void
> +default_diagnostic_final_cb (diagnostic_context *context)
> +{
> +  /* Some of the errors may actually have been warnings.  */
> +  if (diagnostic_kind_count (context, DK_WERROR))
> +    {
> +      /* -Werror was given.  */
> +      if (context->warning_as_error_requested)
> +       pp_verbatim (context->printer,
> +                    _("%s: all warnings being treated as errors"),
> +                    progname);
> +      /* At least one -Werror= was given.  */
> +      else
> +       pp_verbatim (context->printer,
> +                    _("%s: some warnings being treated as errors"),
> +                    progname);
> +      pp_newline_and_flush (context->printer);
> +    }
> +}
> +
>  /* Initialize the diagnostic message outputting machinery.  */
>  void
>  diagnostic_initialize (diagnostic_context *context, int n_opts)
> @@ -185,6 +207,7 @@ diagnostic_initialize (diagnostic_context *context, int n_opts)
>    context->diagnostic_group_emission_count = 0;
>    context->begin_group_cb = NULL;
>    context->end_group_cb = NULL;
> +  context->final_cb = default_diagnostic_final_cb;
>  }
>
>  /* Maybe initialize the color support. We require clients to do this
> @@ -220,21 +243,8 @@ diagnostic_color_init (diagnostic_context *context, int value /*= -1 */)
>  void
>  diagnostic_finish (diagnostic_context *context)
>  {
> -  /* Some of the errors may actually have been warnings.  */
> -  if (diagnostic_kind_count (context, DK_WERROR))
> -    {
> -      /* -Werror was given.  */
> -      if (context->warning_as_error_requested)
> -       pp_verbatim (context->printer,
> -                    _("%s: all warnings being treated as errors"),
> -                    progname);
> -      /* At least one -Werror= was given.  */
> -      else
> -       pp_verbatim (context->printer,
> -                    _("%s: some warnings being treated as errors"),
> -                    progname);
> -      pp_newline_and_flush (context->printer);
> -    }
> +  if (context->final_cb)
> +    context->final_cb (context);
>
>    diagnostic_file_cache_fini ();
>
> diff --git a/gcc/diagnostic.h b/gcc/diagnostic.h
> index 3498a9b..5fa1902 100644
> --- a/gcc/diagnostic.h
> +++ b/gcc/diagnostic.h
> @@ -24,6 +24,17 @@ along with GCC; see the file COPYING3.  If not see
>  #include "pretty-print.h"
>  #include "diagnostic-core.h"
>
> +/* Enum for overriding the standard output format.  */
> +
> +enum diagnostics_output_format
> +{
> +  /* The default: textual output.  */
> +  DIAGNOSTICS_OUTPUT_FORMAT_TEXT,
> +
> +  /* JSON-based output.  */
> +  DIAGNOSTICS_OUTPUT_FORMAT_JSON
> +};
> +
>  /* A diagnostic is described by the MESSAGE to send, the FILE and LINE of
>     its context and its KIND (ice, error, warning, note, ...)  See complete
>     list in diagnostic.def.  */
> @@ -243,6 +254,9 @@ struct diagnostic_context
>    /* If non-NULL, this will be called when a stack of groups is
>       popped if any diagnostics were emitted within that group.  */
>    void (*end_group_cb) (diagnostic_context * context);
> +
> +  /* Callback for final cleanup.  */
> +  void (*final_cb) (diagnostic_context *context);
>  };
>
>  static inline void
> @@ -401,5 +415,7 @@ extern char *file_name_as_prefix (diagnostic_context *, const char *);
>
>  extern char *build_message_string (const char *, ...) ATTRIBUTE_PRINTF_1;
>
> +extern void diagnostic_output_format_init (diagnostic_context *,
> +                                          enum diagnostics_output_format);
>
>  #endif /* ! GCC_DIAGNOSTIC_H */
> diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
> index f26ada0..885eff6 100644
> --- a/gcc/doc/invoke.texi
> +++ b/gcc/doc/invoke.texi
> @@ -268,6 +268,7 @@ Objective-C and Objective-C++ Dialects}.
>  @gccoptlist{-fmessage-length=@var{n}  @gol
>  -fdiagnostics-show-location=@r{[}once@r{|}every-line@r{]}  @gol
>  -fdiagnostics-color=@r{[}auto@r{|}never@r{|}always@r{]}  @gol
> +-fdiagnostics-format=@r{[}text@r{|}json@r{]}  @gol
>  -fno-diagnostics-show-option  -fno-diagnostics-show-caret @gol
>  -fno-diagnostics-show-labels -fno-diagnostics-show-line-numbers @gol
>  -fdiagnostics-minimum-margin-width=@var{width} @gol
> @@ -3948,6 +3949,83 @@ Do not print column numbers in diagnostics.  This may be necessary if
>  diagnostics are being scanned by a program that does not understand the
>  column numbers, such as @command{dejagnu}.
>
> +@item -fdiagnostics-format=@var{FORMAT}
> +@opindex fdiagnostics-format
> +Select a different format for printing diagnostics.
> +@var{FORMAT} is @samp{text} or @samp{json}.
> +The default is @samp{text}.
> +
> +The JSON format consists of a top-level JSON array containing JSON
> +objects representing the diagnostics, where diagnostics can have
> +child diagnostics.
> +
> +For example, this error and note:
> +
> +@smallexample
> +misleading-indentation.c:15:3: warning: this 'if' clause does not
> +  guard... [-Wmisleading-indentation]
> +   15 |   if (flag)
> +      |   ^~
> +misleading-indentation.c:17:5: note: ...this statement, but the latter
> +  is misleadingly indented as if it were guarded by the 'if'
> +   17 |     y = 2;
> +      |     ^
> +@end smallexample
> +
> +@noindent
> +might be printed in JSON form like this:
> +
> +@smallexample
> +[
> +    @{
> +        "children": [
> +            @{
> +                "kind": "note",
> +                "locations": [
> +                    @{
> +                        "caret": @{
> +                            "column": 5,
> +                            "file": "misleading-indentation.c",
> +                            "line": 17
> +                        @}
> +                    @}
> +                ],
> +                "message": "...this statement, but the latter is misleadingly indented as if it were guarded by the \u2018if\u2019"
> +            @}
> +        ],
> +        "kind": "warning",
> +        "locations": [
> +            @{
> +                "caret": @{
> +                    "column": 3,
> +                    "file": "misleading-indentation.c",
> +                    "line": 15
> +                @},
> +                "finish": @{
> +                    "column": 4,
> +                    "file": "misleading-indentation.c",
> +                    "line": 15
> +                @}
> +            @}
> +        ],
> +        "message": "this \u2018if\u2019 clause does not guard...",
> +        "option": "-Wmisleading-indentation"
> +    @},
> +    @dots{}
> +]
> +@end smallexample
> +
> +@noindent
> +where the @code{note} is a child of the @code{warning}.
> +
> +A diagnostic has a @code{kind}.  If this is @code{warning}, then there is
> +an @code{option} key describing the command-line option controlling the
> +warning.
> +
> +A diagnostic can contain zero or more locations.  Each location has up
> +to three positions within it: a @code{caret} position, and, optional
> +@code{start} and @code{finish} positions.
> +
>  @end table
>
>  @node Warning Options
> diff --git a/gcc/dwarf2out.c b/gcc/dwarf2out.c
> index 8b478aa..4fb0527 100644
> --- a/gcc/dwarf2out.c
> +++ b/gcc/dwarf2out.c
> @@ -24267,6 +24267,7 @@ gen_producer_string (void)
>        case OPT_fdiagnostics_show_labels:
>        case OPT_fdiagnostics_show_line_numbers:
>        case OPT_fdiagnostics_color_:
> +      case OPT_fdiagnostics_format_:
>        case OPT_fverbose_asm:
>        case OPT____:
>        case OPT__sysroot_:
> diff --git a/gcc/gcc.c b/gcc/gcc.c
> index c0d17eb..afc18b3 100644
> --- a/gcc/gcc.c
> +++ b/gcc/gcc.c
> @@ -4001,6 +4001,11 @@ driver_handle_option (struct gcc_options *opts,
>        diagnostic_color_init (dc, value);
>        break;
>
> +    case OPT_fdiagnostics_format_:
> +      diagnostic_output_format_init (dc,
> +                                    (enum diagnostics_output_format)value);
> +      break;
> +
>      case OPT_Wa_:
>        {
>         int prev, j;
> diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
> index 2b9d47e..90d8c1c 100644
> --- a/gcc/lto-wrapper.c
> +++ b/gcc/lto-wrapper.c
> @@ -646,6 +646,7 @@ append_diag_options (obstack *argv_obstack, struct cl_decoded_option *opts,
>        switch (option->opt_index)
>         {
>         case OPT_fdiagnostics_color_:
> +       case OPT_fdiagnostics_format_:
>         case OPT_fdiagnostics_show_caret:
>         case OPT_fdiagnostics_show_labels:
>         case OPT_fdiagnostics_show_line_numbers:
> diff --git a/gcc/opts.c b/gcc/opts.c
> index 34c283d..514e7f6 100644
> --- a/gcc/opts.c
> +++ b/gcc/opts.c
> @@ -2220,6 +2220,11 @@ common_handle_option (struct gcc_options *opts,
>        diagnostic_color_init (dc, value);
>        break;
>
> +    case OPT_fdiagnostics_format_:
> +      diagnostic_output_format_init (dc,
> +                                    (enum diagnostics_output_format)value);
> +      break;
> +
>      case OPT_fdiagnostics_parseable_fixits:
>        dc->parseable_fixits_p = value;
>        break;
> --
> 1.8.5.3
>