[gcc r17-169] Introduce pretty-print-token-buffer.{cc,h}
David Malcolm
dmalcolm@gcc.gnu.org
Tue Apr 28 23:12:35 GMT 2026
https://gcc.gnu.org/g:9c529989e0fd9cd4b816cb13cc8e5a99e3cc43c0
commit r17-169-g9c529989e0fd9cd4b816cb13cc8e5a99e3cc43c0
Author: David Malcolm <dmalcolm@redhat.com>
Date: Tue Feb 24 17:44:31 2026 -0500
Introduce pretty-print-token-buffer.{cc,h}
Move the implementation of diagnostic_message_buffer from libdiagnostics
to a new pretty-print-token-buffer.{cc,h}, for capturing the tokens from
a pretty-print.
Implement a new class pp_token_buffer_element for replaying the tokens
in a pretty_print_token_buffer into another pretty-print, using "%e".
Add selftests.
gcc/ChangeLog:
* Makefile.in (OBJS-libcommon): Add pretty-print-token-buffer.o.
* libgdiagnostics.cc: Drop include of "auto-obstack.h".
Include "pretty-print-token-buffer.h".
(class copying_token_printer): Move to
pretty-print-token-buffer.cc.
(struct diagnostic_message_buffer): Reimplement as a subclass of
pretty_print_token_buffer.
(diagnostic_message_buffer::to_string): Rename to
pretty_print_token_buffer::to_string and move to
pretty-print-token-buffer.cc.
* pretty-print-token-buffer.cc: New file, based on material from
libgdiagnostics.cc.
* pretty-print-token-buffer.h: New file, based on material from
libgdiagnostics.h.
* selftest-run-tests.cc (selftest::run_tests): Call
selftest::pretty_print_token_buffer_cc_tests.
* selftest.h (selftest::pretty_print_token_buffer_cc_tests): New
decl.
Signed-off-by: David Malcolm <dmalcolm@redhat.com>
Diff:
---
gcc/Makefile.in | 2 +-
gcc/libgdiagnostics.cc | 177 +------------------
gcc/pretty-print-token-buffer.cc | 362 +++++++++++++++++++++++++++++++++++++++
gcc/pretty-print-token-buffer.h | 70 ++++++++
gcc/selftest-run-tests.cc | 1 +
gcc/selftest.h | 1 +
6 files changed, 440 insertions(+), 173 deletions(-)
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index 8ecef4ccdc7f..bcbcffa4118b 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1904,7 +1904,7 @@ OBJS-libcommon = \
diagnostics/diagnostics-selftests.o \
gcc-diagnostic-spec.o \
graphviz.o pex.o \
- pretty-print.o intl.o \
+ pretty-print.o pretty-print-token-buffer.o intl.o \
json.o json-parsing.o json-diagnostic.o \
pub-sub.o \
xml.o \
diff --git a/gcc/libgdiagnostics.cc b/gcc/libgdiagnostics.cc
index dd5109229b3d..1cf451273d07 100644
--- a/gcc/libgdiagnostics.cc
+++ b/gcc/libgdiagnostics.cc
@@ -44,7 +44,7 @@ along with GCC; see the file COPYING3. If not see
#include "libgdiagnostics-private.h"
#include "pretty-print-format-impl.h"
#include "pretty-print-markup.h"
-#include "auto-obstack.h"
+#include "pretty-print-token-buffer.h"
class owned_nullable_string
{
@@ -252,97 +252,6 @@ private:
diagnostics::source_printing_options m_source_printing;
};
-/* A token_printer that makes a deep copy of the pp_token_list
- into another obstack. */
-
-class copying_token_printer : public token_printer
-{
-public:
- copying_token_printer (obstack &dst_obstack,
- pp_token_list &dst_token_list)
- : m_dst_obstack (dst_obstack),
- m_dst_token_list (dst_token_list)
- {
- }
-
- void
- print_tokens (pretty_printer *,
- const pp_token_list &tokens) final override
- {
- for (auto iter = tokens.m_first; iter; iter = iter->m_next)
- switch (iter->m_kind)
- {
- default:
- gcc_unreachable ();
-
- case pp_token::kind::text:
- {
- const pp_token_text *sub = as_a <const pp_token_text *> (iter);
- /* Copy the text, with null terminator. */
- obstack_grow (&m_dst_obstack, sub->m_value.get (),
- strlen (sub->m_value.get ()) + 1);
- m_dst_token_list.push_back_text
- (label_text::borrow (XOBFINISH (&m_dst_obstack,
- const char *)));
- }
- break;
-
- case pp_token::kind::begin_color:
- {
- pp_token_begin_color *sub = as_a <pp_token_begin_color *> (iter);
- /* Copy the color, with null terminator. */
- obstack_grow (&m_dst_obstack, sub->m_value.get (),
- strlen (sub->m_value.get ()) + 1);
- m_dst_token_list.push_back<pp_token_begin_color>
- (label_text::borrow (XOBFINISH (&m_dst_obstack,
- const char *)));
- }
- break;
- case pp_token::kind::end_color:
- m_dst_token_list.push_back<pp_token_end_color> ();
- break;
-
- case pp_token::kind::begin_quote:
- m_dst_token_list.push_back<pp_token_begin_quote> ();
- break;
- case pp_token::kind::end_quote:
- m_dst_token_list.push_back<pp_token_end_quote> ();
- break;
-
- case pp_token::kind::begin_url:
- {
- pp_token_begin_url *sub = as_a <pp_token_begin_url *> (iter);
- /* Copy the URL, with null terminator. */
- obstack_grow (&m_dst_obstack, sub->m_value.get (),
- strlen (sub->m_value.get ()) + 1);
- m_dst_token_list.push_back<pp_token_begin_url>
- (label_text::borrow (XOBFINISH (&m_dst_obstack,
- const char *)));
- }
- break;
- case pp_token::kind::end_url:
- m_dst_token_list.push_back<pp_token_end_url> ();
- break;
-
- case pp_token::kind::event_id:
- {
- pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
- m_dst_token_list.push_back<pp_token_event_id> (sub->m_event_id);
- }
- break;
-
- case pp_token::kind::custom_data:
- /* These should have been eliminated by replace_custom_tokens. */
- gcc_unreachable ();
- break;
- }
- }
-
-private:
- obstack &m_dst_obstack;
- pp_token_list &m_dst_token_list;
-};
-
class sarif_sink : public sink
{
public:
@@ -352,31 +261,13 @@ public:
const diagnostics::sarif_generation_options &sarif_gen_opts);
};
-struct diagnostic_message_buffer
+struct diagnostic_message_buffer : public pretty_print_token_buffer
{
- diagnostic_message_buffer ()
- : m_tokens (m_obstack)
- {
- }
-
- diagnostic_message_buffer (const char *gmsgid,
- va_list *args)
- : m_tokens (m_obstack)
+ diagnostic_message_buffer () {}
+ diagnostic_message_buffer (const char *gmsgid, va_list *args)
+ : pretty_print_token_buffer (gmsgid, args)
{
- text_info text (gmsgid, args, errno);
- pretty_printer pp;
- pp.set_output_stream (nullptr);
- copying_token_printer tok_printer (m_obstack, m_tokens);
- pp.set_token_printer (&tok_printer);
- pp_format (&pp, &text);
- pp_output_formatted_text (&pp, nullptr);
}
-
-
- std::string to_string () const;
-
- auto_obstack m_obstack;
- pp_token_list m_tokens;
};
/* A pp_element subclass that replays the saved tokens in a
@@ -1566,64 +1457,6 @@ sarif_sink (diagnostic_manager &mgr,
mgr.get_dc ().add_sink (std::move (inner_sink));
}
-// struct diagnostic_message_buffer
-
-std::string
-diagnostic_message_buffer::to_string () const
-{
- std::string result;
-
- /* Convert to text, dropping colorization, URLs, etc. */
- for (auto iter = m_tokens.m_first; iter; iter = iter->m_next)
- switch (iter->m_kind)
- {
- default:
- gcc_unreachable ();
-
- case pp_token::kind::text:
- {
- pp_token_text *sub = as_a <pp_token_text *> (iter);
- result += sub->m_value.get ();
- }
- break;
-
- case pp_token::kind::begin_color:
- case pp_token::kind::end_color:
- // Skip
- break;
-
- case pp_token::kind::begin_quote:
- result += open_quote;
- break;
-
- case pp_token::kind::end_quote:
- result += close_quote;
- break;
-
- case pp_token::kind::begin_url:
- case pp_token::kind::end_url:
- // Skip
- break;
-
- case pp_token::kind::event_id:
- {
- pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
- gcc_assert (sub->m_event_id.known_p ());
- result += '(';
- result += std::to_string (sub->m_event_id.one_based ());
- result += ')';
- }
- break;
-
- case pp_token::kind::custom_data:
- /* We don't have a way of handling custom_data tokens here. */
- gcc_unreachable ();
- break;
- }
-
- return result;
-}
-
/* struct diagnostic_manager. */
void
diff --git a/gcc/pretty-print-token-buffer.cc b/gcc/pretty-print-token-buffer.cc
new file mode 100644
index 000000000000..61c4a7608b3c
--- /dev/null
+++ b/gcc/pretty-print-token-buffer.cc
@@ -0,0 +1,362 @@
+/* Capturing the results of pretty_print for later playback.
+ Copyright (C) 2023-2026 Free Software Foundation, Inc.
+ Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#include "config.h"
+#define INCLUDE_STRING
+#include "system.h"
+#include "coretypes.h"
+#include "intl.h"
+#include "pretty-print.h"
+#include "pretty-print-token-buffer.h"
+#include "selftest.h"
+
+/* A token_printer that makes a deep copy of the pp_token_list
+ into another obstack. */
+
+class copying_token_printer : public token_printer
+{
+public:
+ copying_token_printer (obstack &dst_obstack,
+ pp_token_list &dst_token_list)
+ : m_dst_obstack (dst_obstack),
+ m_dst_token_list (dst_token_list)
+ {
+ }
+
+ void
+ print_tokens (pretty_printer *,
+ const pp_token_list &tokens) final override
+ {
+ for (auto iter = tokens.m_first; iter; iter = iter->m_next)
+ switch (iter->m_kind)
+ {
+ default:
+ gcc_unreachable ();
+
+ case pp_token::kind::text:
+ {
+ const pp_token_text *sub = as_a <const pp_token_text *> (iter);
+ /* Copy the text, with null terminator. */
+ obstack_grow (&m_dst_obstack, sub->m_value.get (),
+ strlen (sub->m_value.get ()) + 1);
+ m_dst_token_list.push_back_text
+ (label_text::borrow (XOBFINISH (&m_dst_obstack,
+ const char *)));
+ }
+ break;
+
+ case pp_token::kind::begin_color:
+ {
+ pp_token_begin_color *sub = as_a <pp_token_begin_color *> (iter);
+ /* Copy the color, with null terminator. */
+ obstack_grow (&m_dst_obstack, sub->m_value.get (),
+ strlen (sub->m_value.get ()) + 1);
+ m_dst_token_list.push_back<pp_token_begin_color>
+ (label_text::borrow (XOBFINISH (&m_dst_obstack,
+ const char *)));
+ }
+ break;
+ case pp_token::kind::end_color:
+ m_dst_token_list.push_back<pp_token_end_color> ();
+ break;
+
+ case pp_token::kind::begin_quote:
+ m_dst_token_list.push_back<pp_token_begin_quote> ();
+ break;
+ case pp_token::kind::end_quote:
+ m_dst_token_list.push_back<pp_token_end_quote> ();
+ break;
+
+ case pp_token::kind::begin_url:
+ {
+ pp_token_begin_url *sub = as_a <pp_token_begin_url *> (iter);
+ /* Copy the URL, with null terminator. */
+ obstack_grow (&m_dst_obstack, sub->m_value.get (),
+ strlen (sub->m_value.get ()) + 1);
+ m_dst_token_list.push_back<pp_token_begin_url>
+ (label_text::borrow (XOBFINISH (&m_dst_obstack,
+ const char *)));
+ }
+ break;
+ case pp_token::kind::end_url:
+ m_dst_token_list.push_back<pp_token_end_url> ();
+ break;
+
+ case pp_token::kind::event_id:
+ {
+ pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+ m_dst_token_list.push_back<pp_token_event_id> (sub->m_event_id);
+ }
+ break;
+
+ case pp_token::kind::custom_data:
+ /* These should have been eliminated by replace_custom_tokens. */
+ gcc_unreachable ();
+ break;
+ }
+ }
+
+private:
+ obstack &m_dst_obstack;
+ pp_token_list &m_dst_token_list;
+};
+
+pretty_print_token_buffer::pretty_print_token_buffer ()
+: m_obstack (std::make_unique<auto_obstack> ()),
+ m_tokens (*m_obstack.get ())
+{
+}
+
+/* Capture GMSGID and ARGS as a sequence of pretty_print tokens. */
+
+pretty_print_token_buffer::pretty_print_token_buffer (const char *gmsgid,
+ va_list *args)
+: m_obstack (std::make_unique<auto_obstack> ()),
+ m_tokens (*m_obstack.get ())
+{
+ text_info text (gmsgid, args, errno);
+ pretty_printer pp;
+ pp.set_output_stream (nullptr);
+ copying_token_printer tok_printer (*m_obstack.get (), m_tokens);
+ pp.set_token_printer (&tok_printer);
+ pp_format (&pp, &text);
+ pp_output_formatted_text (&pp, nullptr);
+}
+
+pretty_print_token_buffer::
+pretty_print_token_buffer (pretty_print_token_buffer &&other)
+: m_obstack (std::move (other.m_obstack)),
+ m_tokens (std::move (other.m_tokens))
+{
+}
+
+/* Convert to text, dropping colorization, URLs, etc. */
+
+std::string
+pretty_print_token_buffer::to_string () const
+{
+ std::string result;
+
+ for (auto iter = m_tokens.m_first; iter; iter = iter->m_next)
+ switch (iter->m_kind)
+ {
+ default:
+ gcc_unreachable ();
+
+ case pp_token::kind::text:
+ {
+ pp_token_text *sub = as_a <pp_token_text *> (iter);
+ result += sub->m_value.get ();
+ }
+ break;
+
+ case pp_token::kind::begin_color:
+ case pp_token::kind::end_color:
+ // Skip
+ break;
+
+ case pp_token::kind::begin_quote:
+ result += open_quote;
+ break;
+
+ case pp_token::kind::end_quote:
+ result += close_quote;
+ break;
+
+ case pp_token::kind::begin_url:
+ case pp_token::kind::end_url:
+ // Skip
+ break;
+
+ case pp_token::kind::event_id:
+ {
+ pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+ gcc_assert (sub->m_event_id.known_p ());
+ result += '(';
+ result += std::to_string (sub->m_event_id.one_based ());
+ result += ')';
+ }
+ break;
+
+ case pp_token::kind::custom_data:
+ /* We don't have a way of handling custom_data tokens here. */
+ gcc_unreachable ();
+ break;
+ }
+
+ return result;
+}
+
+// class pp_token_buffer_element : public pp_element
+
+void
+pp_token_buffer_element::add_to_phase_2 (pp_markup::context &ctxt)
+{
+ for (auto iter = m_token_buf.m_tokens.m_first; iter; iter = iter->m_next)
+ switch (iter->m_kind)
+ {
+ default:
+ gcc_unreachable ();
+
+ case pp_token::kind::text:
+ {
+ const pp_token_text *sub = as_a <const pp_token_text *> (iter);
+ pp_string (&ctxt.m_pp, sub->m_value.get ());
+ ctxt.push_back_any_text ();
+ }
+ break;
+
+ case pp_token::kind::begin_color:
+ {
+ const pp_token_begin_color *sub
+ = as_a <const pp_token_begin_color *> (iter);
+ gcc_assert (sub->m_value.get ());
+ ctxt.begin_highlight_color (sub->m_value.get ());
+ }
+ break;
+
+ case pp_token::kind::end_color:
+ ctxt.end_highlight_color ();
+ break;
+
+ case pp_token::kind::begin_quote:
+ ctxt.begin_quote ();
+ break;
+
+ case pp_token::kind::end_quote:
+ ctxt.end_quote ();
+ break;
+
+ case pp_token::kind::begin_url:
+ {
+ const pp_token_begin_url *sub
+ = as_a <const pp_token_begin_url *> (iter);
+ gcc_assert (sub->m_value.get ());
+ ctxt.begin_url (sub->m_value.get ());
+ }
+ break;
+
+ case pp_token::kind::end_url:
+ ctxt.end_url ();
+ break;
+
+ case pp_token::kind::event_id:
+ {
+ const pp_token_event_id *sub
+ = as_a <const pp_token_event_id *> (iter);
+ gcc_assert (sub->m_event_id.known_p ());
+ ctxt.add_event_id (sub->m_event_id);
+ }
+ break;
+
+ case pp_token::kind::custom_data:
+ /* We don't have a way of handling custom_data tokens here. */
+ gcc_unreachable ();
+ break;
+ }
+}
+
+#if CHECKING_P
+
+namespace selftest {
+
+static pretty_print_token_buffer
+pp_printf_to_buf (const char *fmt, ...)
+{
+ va_list args;
+ va_start (args, fmt);
+
+ pretty_print_token_buffer buf (fmt, &args);
+
+ va_end (args);
+
+ return buf;
+}
+
+static void
+test_empty ()
+{
+ pretty_print_token_buffer buf;
+ pp_token_buffer_element e (buf);
+ pretty_printer pp;
+ pp_printf (&pp, "before %e after", &e);
+ ASSERT_STREQ (pp_formatted_text (&pp), "before after");
+}
+
+static void
+test_print ()
+{
+ pretty_print_token_buffer buf
+ = pp_printf_to_buf ("x: %qs y: %qs", "foo", "bar");
+
+ // Check that the individual tokens are captured in "buf".
+ pp_token *tok0 = buf.m_tokens.m_first;
+ ASSERT_EQ (tok0->m_kind, pp_token::kind::text);
+ ASSERT_STREQ (((pp_token_text *)tok0)->m_value.get (),
+ "x: ");
+
+ pp_token *tok1 = tok0->m_next;
+ ASSERT_EQ (tok1->m_kind, pp_token::kind::begin_quote);
+
+ pp_token *tok2 = tok1->m_next;
+ ASSERT_EQ (tok2->m_kind, pp_token::kind::text);
+ ASSERT_STREQ (((pp_token_text *)tok2)->m_value.get (),
+ "foo");
+
+ pp_token *tok3 = tok2->m_next;
+ ASSERT_EQ (tok3->m_kind, pp_token::kind::end_quote);
+
+ pp_token *tok4 = tok3->m_next;
+ ASSERT_EQ (tok4->m_kind, pp_token::kind::text);
+ ASSERT_STREQ (((pp_token_text *)tok4)->m_value.get (),
+ " y: ");
+
+ pp_token *tok5 = tok4->m_next;
+ ASSERT_EQ (tok5->m_kind, pp_token::kind::begin_quote);
+
+ pp_token *tok6 = tok5->m_next;
+ ASSERT_EQ (tok6->m_kind, pp_token::kind::text);
+ ASSERT_STREQ (((pp_token_text *)tok6)->m_value.get (),
+ "bar");
+
+ pp_token *tok7 = tok6->m_next;
+ ASSERT_EQ (tok7->m_kind, pp_token::kind::end_quote);
+ ASSERT_EQ (tok7->m_next, nullptr);
+
+
+ // Check that we can replay buf via pp_token_buffer_element
+ pp_token_buffer_element e (buf);
+ pretty_printer pp;
+ pp_printf (&pp, "before %e after", &e);
+ ASSERT_STREQ (pp_formatted_text (&pp), "before x: 'foo' y: 'bar' after");
+}
+
+/* Run all of the selftests within this file. */
+
+void
+pretty_print_token_buffer_cc_tests ()
+{
+ test_empty ();
+ test_print ();
+}
+
+} // namespace selftest
+
+#endif /* CHECKING_P */
diff --git a/gcc/pretty-print-token-buffer.h b/gcc/pretty-print-token-buffer.h
new file mode 100644
index 000000000000..cab691d2c3f7
--- /dev/null
+++ b/gcc/pretty-print-token-buffer.h
@@ -0,0 +1,70 @@
+/* Capturing the results of pretty_print for later playback.
+ Copyright (C) 2023-2026 Free Software Foundation, Inc.
+ Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#ifndef GCC_PRETTY_PRINT_TOKEN_BUFFER_H
+#define GCC_PRETTY_PRINT_TOKEN_BUFFER_H
+
+#include "pretty-print-format-impl.h"
+#include "auto-obstack.h"
+#include "pretty-print-markup.h"
+
+/* A class for capturing the results of pretty-printing as tokens,
+ potentially for playback into a different pretty-printer. */
+
+class pretty_print_token_buffer
+{
+public:
+ pretty_print_token_buffer ();
+ pretty_print_token_buffer (const char *gmsgid,
+ va_list *args);
+
+ pretty_print_token_buffer (const pretty_print_token_buffer &) = delete;
+ pretty_print_token_buffer (pretty_print_token_buffer &&);
+
+ ~pretty_print_token_buffer () = default;
+
+ std::string to_string () const;
+
+ void dump (FILE *out) const { m_tokens.dump (out); }
+ void DEBUG_FUNCTION dump () const { dump (stderr); }
+
+ std::unique_ptr<auto_obstack> m_obstack;
+ pp_token_list m_tokens;
+};
+
+/* A pp_element subclass for use with "%e" that replays the buffered tokens
+ from TOKEN_BUF in another formatting call. */
+
+class pp_token_buffer_element : public pp_element
+{
+public:
+ pp_token_buffer_element (const pretty_print_token_buffer &token_buf)
+ : m_token_buf (token_buf)
+ {
+ }
+
+ void
+ add_to_phase_2 (pp_markup::context &ctxt) final override;
+
+private:
+ const pretty_print_token_buffer &m_token_buf;
+};
+
+#endif /* GCC_PRETTY_PRINT_TOKEN_BUFFER_H */
diff --git a/gcc/selftest-run-tests.cc b/gcc/selftest-run-tests.cc
index 7ead010360f9..86b2340e9b93 100644
--- a/gcc/selftest-run-tests.cc
+++ b/gcc/selftest-run-tests.cc
@@ -68,6 +68,7 @@ selftest::run_tests ()
hash_set_tests_cc_tests ();
vec_cc_tests ();
pretty_print_cc_tests ();
+ pretty_print_token_buffer_cc_tests ();
wide_int_cc_tests ();
ggc_tests_cc_tests ();
sreal_cc_tests ();
diff --git a/gcc/selftest.h b/gcc/selftest.h
index 462786dab2f5..e3e86ffaf0f4 100644
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -246,6 +246,7 @@ extern void ordered_hash_map_tests_cc_tests ();
extern void path_coverage_cc_tests ();
extern void predict_cc_tests ();
extern void pretty_print_cc_tests ();
+extern void pretty_print_token_buffer_cc_tests ();
extern void pub_sub_cc_tests ();
extern void range_op_tests ();
extern void range_tests ();
More information about the Gcc-cvs
mailing list