[PATCH Rust front-end v2 09/37] gccrs: Add Lexer for Rust front-end

herron.philip@googlemail.com herron.philip@googlemail.com
Wed Aug 24 11:59:28 GMT 2022


From: The Other <simplytheother@gmail.com>

The lexer is refered to as a ManagedTokenSource within the parser, this
lexer does not currently support unicode but serves as a starting point
to do so.

Co-authored-by: Philip Herron <philip.herron@embecosm.com>
Co-authored-by: Arthur Cohen <arthur.cohen@embecosm.com>
Co-authored-by: Mark Wielaard <mark@klomp.org>
---
 gcc/rust/lex/rust-codepoint.h  |   46 +
 gcc/rust/lex/rust-lex.cc       | 2729 ++++++++++++++++++++++++++++++++
 gcc/rust/lex/rust-lex.h        |  271 ++++
 gcc/rust/lex/rust-token.cc     |  135 ++
 gcc/rust/lex/rust-token.h      |  455 ++++++
 gcc/rust/rust-buffered-queue.h |  204 +++
 6 files changed, 3840 insertions(+)
 create mode 100644 gcc/rust/lex/rust-codepoint.h
 create mode 100644 gcc/rust/lex/rust-lex.cc
 create mode 100644 gcc/rust/lex/rust-lex.h
 create mode 100644 gcc/rust/lex/rust-token.cc
 create mode 100644 gcc/rust/lex/rust-token.h
 create mode 100644 gcc/rust/rust-buffered-queue.h

diff --git a/gcc/rust/lex/rust-codepoint.h b/gcc/rust/lex/rust-codepoint.h
new file mode 100644
index 00000000000..22da080bbb2
--- /dev/null
+++ b/gcc/rust/lex/rust-codepoint.h
@@ -0,0 +1,46 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_CODEPOINT_H
+#define RUST_CODEPOINT_H
+
+#include <string>
+
+namespace Rust {
+struct Codepoint
+{
+  uint32_t value;
+
+  // Creates a zero codepoint.
+  Codepoint () : value (0) {}
+
+  // Creates a codepoint from an encoded UTF-8 value.
+  Codepoint (uint32_t value) : value (value) {}
+
+  static Codepoint eof () { return Codepoint (UINT32_MAX); }
+  bool is_eof () const { return value == UINT32_MAX; }
+
+  // Returns a C++ string containing string value of codepoint.
+  std::string as_string ();
+
+  bool operator== (Codepoint other) const { return value == other.value; }
+  bool operator!= (Codepoint other) const { return !operator== (other); }
+};
+} // namespace Rust
+
+#endif
diff --git a/gcc/rust/lex/rust-lex.cc b/gcc/rust/lex/rust-lex.cc
new file mode 100644
index 00000000000..70e6b50209f
--- /dev/null
+++ b/gcc/rust/lex/rust-lex.cc
@@ -0,0 +1,2729 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-lex.h"
+
+#include "rust-system.h"      // for rust_assert and rust_unreachable
+#include "rust-diagnostics.h" // for rust_error_at
+#include "rust-linemap.h"
+#include "rust-session-manager.h"
+#include "safe-ctype.h"
+
+namespace Rust {
+// TODO: move to separate compilation unit?
+// overload += for uint32_t to allow 32-bit encoded utf-8 to be added
+std::string &
+operator+= (std::string &str, Codepoint char32)
+{
+  if (char32.value < 0x80)
+    {
+      str += static_cast<char> (char32.value);
+    }
+  else if (char32.value < (0x1F + 1) << (1 * 6))
+    {
+      str += static_cast<char> (0xC0 | ((char32.value >> 6) & 0x1F));
+      str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
+    }
+  else if (char32.value < (0x0F + 1) << (2 * 6))
+    {
+      str += static_cast<char> (0xE0 | ((char32.value >> 12) & 0x0F));
+      str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
+      str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
+    }
+  else if (char32.value < (0x07 + 1) << (3 * 6))
+    {
+      str += static_cast<char> (0xF0 | ((char32.value >> 18) & 0x07));
+      str += static_cast<char> (0x80 | ((char32.value >> 12) & 0x3F));
+      str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
+      str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
+    }
+  else
+    {
+      rust_debug ("Invalid unicode codepoint found: '%u' ", char32.value);
+    }
+  return str;
+}
+
+std::string
+Codepoint::as_string ()
+{
+  std::string str;
+
+  // str += Codepoint (value);
+  str += *this;
+
+  return str;
+}
+
+/* Includes all allowable float digits EXCEPT _ and . as that needs lookahead
+ * for handling. */
+bool
+is_float_digit (char number)
+{
+  return ISDIGIT (number) || number == 'E' || number == 'e';
+}
+
+/* Basically ISXDIGIT from safe-ctype but may change if Rust's encoding or
+ * whatever is different */
+bool
+is_x_digit (char number)
+{
+  return ISXDIGIT (number);
+}
+
+bool
+is_octal_digit (char number)
+{
+  return number >= '0' && number <= '7';
+}
+
+bool
+is_bin_digit (char number)
+{
+  return number == '0' || number == '1';
+}
+
+bool
+check_valid_float_dot_end (char character)
+{
+  return character != '.' && character != '_' && !ISALPHA (character);
+}
+
+// ISSPACE from safe-ctype but may change in future
+bool
+is_whitespace (char character)
+{
+  return ISSPACE (character);
+}
+
+bool
+is_non_decimal_int_literal_separator (char character)
+{
+  return character == 'x' || character == 'o' || character == 'b';
+}
+
+Lexer::Lexer (const std::string &input)
+  : input (RAIIFile::create_error ()), current_line (1), current_column (1),
+    line_map (nullptr), raw_input_source (new BufferInputSource (input, 0)),
+    input_queue{*raw_input_source}, token_queue (TokenSource (this))
+{}
+
+Lexer::Lexer (const char *filename, RAIIFile file_input, Linemap *linemap)
+  : input (std::move (file_input)), current_line (1), current_column (1),
+    line_map (linemap),
+    raw_input_source (new FileInputSource (input.get_raw ())),
+    input_queue{*raw_input_source}, token_queue (TokenSource (this))
+{
+  // inform line_table that file is being entered and is in line 1
+  if (linemap)
+    line_map->start_file (filename, current_line);
+}
+
+Lexer::~Lexer ()
+{
+  /* ok apparently stop (which is equivalent of original code in destructor) is
+   * meant to be called after all files have finished parsing, for cleanup. On
+   * the other hand, actual code that it calls to leave a certain line map is
+   * mentioned in GCC docs as being useful for "just leaving an included header"
+   * and stuff like that, so this line mapping functionality may need fixing.
+   * FIXME: find out whether this occurs. */
+
+  // line_map->stop();
+}
+
+/* TODO: need to optimise somehow to avoid the virtual function call in the
+ * tight loop. Best idea at the moment is CRTP, but that might make lexer
+ * implementation annoying when storing the "base class" (i.e. would need
+ * template parameter everywhere), although in practice it would mostly just
+ * look ugly and make enclosing classes like Parser also require a type
+ * parameter. At this point a macro might be better. OK I guess macros can be
+ * replaced by constexpr if or something if possible. */
+Location
+Lexer::get_current_location ()
+{
+  if (line_map)
+    return line_map->get_location (current_column);
+  else
+    // If we have no linemap, we're lexing something without proper locations
+    return Location ();
+}
+
+int
+Lexer::peek_input (int n)
+{
+  return input_queue.peek (n);
+}
+
+int
+Lexer::peek_input ()
+{
+  return peek_input (0);
+}
+
+void
+Lexer::skip_input (int n)
+{
+  input_queue.skip (n);
+}
+
+void
+Lexer::skip_input ()
+{
+  skip_input (0);
+}
+
+void
+Lexer::replace_current_token (TokenPtr replacement)
+{
+  token_queue.replace_current_value (replacement);
+
+  rust_debug ("called 'replace_current_token' - this is deprecated");
+}
+
+/* shitty anonymous namespace that can only be accessed inside the compilation
+ * unit - used for classify_keyword binary search in sorted array of keywords
+ * created with x-macros. */
+namespace {
+// TODO: make constexpr when update to c++20
+const std::string keyword_index[] = {
+#define RS_TOKEN(x, y)
+#define RS_TOKEN_KEYWORD(name, keyword) keyword,
+  RS_TOKEN_LIST
+#undef RS_TOKEN_KEYWORD
+#undef RS_TOKEN
+};
+
+constexpr TokenId keyword_keys[] = {
+#define RS_TOKEN(x, y)
+#define RS_TOKEN_KEYWORD(name, keyword) name,
+  RS_TOKEN_LIST
+#undef RS_TOKEN_KEYWORD
+#undef RS_TOKEN
+};
+
+constexpr int num_keywords = sizeof (keyword_index) / sizeof (*keyword_index);
+} // namespace
+
+/* Determines whether the string passed in is a keyword or not. If it is, it
+ * returns the keyword name.  */
+TokenId
+Lexer::classify_keyword (const std::string &str)
+{
+  const std::string *last = keyword_index + num_keywords;
+  const std::string *idx = std::lower_bound (keyword_index, last, str);
+
+  if (idx == last || str != *idx)
+    return IDENTIFIER;
+
+  // TODO: possibly replace this x-macro system with something like hash map?
+
+  // We now have the expected token ID of the reserved keyword. However, some
+  // keywords are reserved starting in certain editions. For example, `try` is
+  // only a reserved keyword in editions >=2018. The language might gain new
+  // reserved keywords in the future.
+  //
+  // https://doc.rust-lang.org/reference/keywords.html#reserved-keywords
+  auto id = keyword_keys[idx - keyword_index];
+
+  // `try` is not a reserved keyword before 2018
+  if (Session::get_instance ().options.get_edition ()
+	== CompileOptions::Edition::E2015
+      && id == TRY)
+    return IDENTIFIER;
+
+  return id;
+}
+
+TokenPtr
+Lexer::build_token ()
+{
+  // loop to go through multiple characters to build a single token
+  while (true)
+    {
+      Location loc = get_current_location ();
+      current_char = peek_input ();
+      skip_input ();
+
+      // detect UTF8 bom
+      //
+      // Must be the first thing on the first line.
+      // There might be an optional BOM (Byte Order Mark), which for UTF-8 is
+      // the three bytes 0xEF, 0xBB and 0xBF. These can simply be skipped.
+      if (current_line == 1 && current_column == 1 && current_char == 0xef
+	  && peek_input () == 0xbb && peek_input (1) == 0xbf)
+	{
+	  skip_input (1);
+	  current_char = peek_input ();
+	  skip_input ();
+	}
+
+      // detect shebang
+      // Must be the first thing on the first line, starting with #!
+      // But since an attribute can also start with an #! we don't count it as a
+      // shebang line when after any whitespace or comments there is a [. If it
+      // is a shebang line we simple drop the line. Otherwise we don't consume
+      // any characters and fall through to the real tokenizer.
+      if (current_line == 1 && current_column == 1 && current_char == '#'
+	  && peek_input () == '!')
+	{
+	  int n = 1;
+	  while (true)
+	    {
+	      int next_char = peek_input (n);
+	      if (is_whitespace (next_char))
+		n++;
+	      else if ((next_char == '/' && peek_input (n + 1) == '/'
+			&& peek_input (n + 2) != '!'
+			&& peek_input (n + 2) != '/')
+		       || (next_char == '/' && peek_input (n + 1) == '/'
+			   && peek_input (n + 2) == '/'
+			   && peek_input (n + 3) == '/'))
+		{
+		  // two // or four ////
+		  // A single line comment
+		  // (but not an inner or outer doc comment)
+		  n += 2;
+		  next_char = peek_input (n);
+		  while (next_char != '\n' && next_char != EOF)
+		    {
+		      n++;
+		      next_char = peek_input (n);
+		    }
+		  if (next_char == '\n')
+		    n++;
+		}
+	      else if (next_char == '/' && peek_input (n + 1) == '*'
+		       && peek_input (n + 2) == '*'
+		       && peek_input (n + 3) == '/')
+		{
+		  /**/
+		  n += 4;
+		}
+	      else if (next_char == '/' && peek_input (n + 1) == '*'
+		       && peek_input (n + 2) == '*' && peek_input (n + 3) == '*'
+		       && peek_input (n + 4) == '/')
+		{
+		  /***/
+		  n += 5;
+		}
+	      else if ((next_char == '/' && peek_input (n + 1) == '*'
+			&& peek_input (n + 2) != '*'
+			&& peek_input (n + 2) != '!')
+		       || (next_char == '/' && peek_input (n + 1) == '*'
+			   && peek_input (n + 2) == '*'
+			   && peek_input (n + 3) == '*'))
+		{
+		  // one /* or three /***
+		  // Start of a block comment
+		  // (but not an inner or outer doc comment)
+		  n += 2;
+		  int level = 1;
+		  while (level > 0)
+		    {
+		      if (peek_input (n) == EOF)
+			break;
+		      else if (peek_input (n) == '/'
+			       && peek_input (n + 1) == '*')
+			{
+			  n += 2;
+			  level += 1;
+			}
+		      else if (peek_input (n) == '*'
+			       && peek_input (n + 1) == '/')
+			{
+			  n += 2;
+			  level -= 1;
+			}
+		      else
+			n++;
+		    }
+		}
+	      else if (next_char != '[')
+		{
+		  // definitely shebang, ignore the first line
+		  while (current_char != '\n' && current_char != EOF)
+		    {
+		      current_char = peek_input ();
+		      skip_input ();
+		    }
+
+		  // newline
+		  current_line++;
+		  current_column = 1;
+		  // tell line_table that new line starts
+		  start_line (current_line, max_column_hint);
+		  break;
+		}
+	      else
+		break; /* Definitely not a shebang line. */
+	    }
+	}
+
+      // return end of file token if end of file
+      if (current_char == EOF)
+	return Token::make (END_OF_FILE, loc);
+
+      // if not end of file, start tokenising
+      switch (current_char)
+	{
+	/* ignore whitespace characters for tokens but continue updating
+	 * location */
+	case '\n': // newline
+	  current_line++;
+	  current_column = 1;
+	  // tell line_table that new line starts
+	  start_line (current_line, max_column_hint);
+	  continue;
+	case '\r': // cr
+	  // Ignore, we expect a newline (lf) soon.
+	  continue;
+	case ' ': // space
+	  current_column++;
+	  continue;
+	case '\t': // tab
+	  // width of a tab is not well-defined, assume 8 spaces
+	  current_column += 8;
+	  continue;
+
+	// punctuation - actual tokens
+	case '=':
+	  if (peek_input () == '>')
+	    {
+	      // match arm arrow
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (MATCH_ARROW, loc);
+	    }
+	  else if (peek_input () == '=')
+	    {
+	      // equality operator
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (EQUAL_EQUAL, loc);
+	    }
+	  else
+	    {
+	      // assignment operator
+	      current_column++;
+	      return Token::make (EQUAL, loc);
+	    }
+	case '(':
+	  current_column++;
+	  return Token::make (LEFT_PAREN, loc);
+	case '-':
+	  if (peek_input () == '>')
+	    {
+	      // return type specifier
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (RETURN_TYPE, loc);
+	    }
+	  else if (peek_input () == '=')
+	    {
+	      // minus-assign
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (MINUS_EQ, loc);
+	    }
+	  else
+	    {
+	      // minus
+	      current_column++;
+	      return Token::make (MINUS, loc);
+	    }
+	case '+':
+	  if (peek_input () == '=')
+	    {
+	      // add-assign
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (PLUS_EQ, loc);
+	    }
+	  else
+	    {
+	      // add
+	      current_column++;
+	      return Token::make (PLUS, loc);
+	    }
+	case ')':
+	  current_column++;
+	  return Token::make (RIGHT_PAREN, loc);
+	case ';':
+	  current_column++;
+	  return Token::make (SEMICOLON, loc);
+	case '*':
+	  if (peek_input () == '=')
+	    {
+	      // multiplication-assign
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (ASTERISK_EQ, loc);
+	    }
+	  else
+	    {
+	      // multiplication
+	      current_column++;
+	      return Token::make (ASTERISK, loc);
+	    }
+	case ',':
+	  current_column++;
+	  return Token::make (COMMA, loc);
+	case '/':
+	  if (peek_input () == '=')
+	    {
+	      // division-assign
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (DIV_EQ, loc);
+	    }
+	  else if ((peek_input () == '/' && peek_input (1) != '!'
+		    && peek_input (1) != '/')
+		   || (peek_input () == '/' && peek_input (1) == '/'
+		       && peek_input (2) == '/'))
+	    {
+	      // two // or four ////
+	      // single line comment
+	      // (but not an inner or outer doc comment)
+	      skip_input ();
+	      current_column += 2;
+	      current_char = peek_input ();
+
+	      // basically ignore until line finishes
+	      while (current_char != '\n' && current_char != EOF)
+		{
+		  skip_input ();
+		  current_column++; // not used
+		  current_char = peek_input ();
+		}
+	      continue;
+	    }
+	  else if (peek_input () == '/'
+		   && (peek_input (1) == '!' || peek_input (1) == '/'))
+	    {
+	      /* single line doc comment, inner or outer.  */
+	      bool is_inner = peek_input (1) == '!';
+	      skip_input (1);
+	      current_column += 3;
+
+	      std::string str;
+	      str.reserve (32);
+	      current_char = peek_input ();
+	      while (current_char != '\n')
+		{
+		  skip_input ();
+		  if (current_char == '\r')
+		    {
+		      char next_char = peek_input ();
+		      if (next_char == '\n')
+			{
+			  current_char = '\n';
+			  break;
+			}
+		      rust_error_at (
+			loc, "Isolated CR %<\\r%> not allowed in doc comment");
+		      current_char = next_char;
+		      continue;
+		    }
+		  if (current_char == EOF)
+		    {
+		      rust_error_at (
+			loc, "unexpected EOF while looking for end of comment");
+		      break;
+		    }
+		  str += current_char;
+		  current_char = peek_input ();
+		}
+	      skip_input ();
+	      current_line++;
+	      current_column = 1;
+	      // tell line_table that new line starts
+	      start_line (current_line, max_column_hint);
+
+	      str.shrink_to_fit ();
+	      if (is_inner)
+		return Token::make_inner_doc_comment (loc, std::move (str));
+	      else
+		return Token::make_outer_doc_comment (loc, std::move (str));
+	    }
+	  else if (peek_input () == '*' && peek_input (1) == '*'
+		   && peek_input (2) == '/')
+	    {
+	      /**/
+	      skip_input (2);
+	      current_column += 4;
+	      continue;
+	    }
+	  else if (peek_input () == '*' && peek_input (1) == '*'
+		   && peek_input (2) == '*' && peek_input (3) == '/')
+	    {
+	      /***/
+	      skip_input (3);
+	      current_column += 5;
+	      continue;
+	    }
+	  else if ((peek_input () == '*' && peek_input (1) != '!'
+		    && peek_input (1) != '*')
+		   || (peek_input () == '*' && peek_input (1) == '*'
+		       && peek_input (2) == '*'))
+	    {
+	      // one /* or three /***
+	      // block comment
+	      // (but not an inner or outer doc comment)
+	      skip_input ();
+	      current_column += 2;
+
+	      int level = 1;
+	      while (level > 0)
+		{
+		  current_char = peek_input ();
+
+		  if (current_char == EOF)
+		    {
+		      rust_error_at (
+			loc, "unexpected EOF while looking for end of comment");
+		      break;
+		    }
+
+		  // if /* found
+		  if (current_char == '/' && peek_input (1) == '*')
+		    {
+		      // skip /* characters
+		      skip_input (1);
+
+		      current_column += 2;
+
+		      level += 1;
+		      continue;
+		    }
+
+		  // ignore until */ is found
+		  if (current_char == '*' && peek_input (1) == '/')
+		    {
+		      // skip */ characters
+		      skip_input (1);
+
+		      current_column += 2;
+
+		      level -= 1;
+		      continue;
+		    }
+
+		  if (current_char == '\n')
+		    {
+		      skip_input ();
+		      current_line++;
+		      current_column = 1;
+		      // tell line_table that new line starts
+		      start_line (current_line, max_column_hint);
+		      continue;
+		    }
+
+		  skip_input ();
+		  current_column++;
+		}
+
+	      // refresh new token
+	      continue;
+	    }
+	  else if (peek_input () == '*'
+		   && (peek_input (1) == '!' || peek_input (1) == '*'))
+	    {
+	      // block doc comment, inner /*! or outer /**
+	      bool is_inner = peek_input (1) == '!';
+	      skip_input (1);
+	      current_column += 3;
+
+	      std::string str;
+	      str.reserve (96);
+
+	      int level = 1;
+	      while (level > 0)
+		{
+		  current_char = peek_input ();
+
+		  if (current_char == EOF)
+		    {
+		      rust_error_at (
+			loc, "unexpected EOF while looking for end of comment");
+		      break;
+		    }
+
+		  // if /* found
+		  if (current_char == '/' && peek_input (1) == '*')
+		    {
+		      // skip /* characters
+		      skip_input (1);
+		      current_column += 2;
+
+		      level += 1;
+		      str += "/*";
+		      continue;
+		    }
+
+		  // ignore until */ is found
+		  if (current_char == '*' && peek_input (1) == '/')
+		    {
+		      // skip */ characters
+		      skip_input (1);
+		      current_column += 2;
+
+		      level -= 1;
+		      if (level > 0)
+			str += "*/";
+		      continue;
+		    }
+
+		  if (current_char == '\r' && peek_input (1) != '\n')
+		    rust_error_at (
+		      loc, "Isolated CR %<\\r%> not allowed in doc comment");
+
+		  if (current_char == '\n')
+		    {
+		      skip_input ();
+		      current_line++;
+		      current_column = 1;
+		      // tell line_table that new line starts
+		      start_line (current_line, max_column_hint);
+		      str += '\n';
+		      continue;
+		    }
+
+		  str += current_char;
+		  skip_input ();
+		  current_column++;
+		}
+
+	      str.shrink_to_fit ();
+	      if (is_inner)
+		return Token::make_inner_doc_comment (loc, std::move (str));
+	      else
+		return Token::make_outer_doc_comment (loc, std::move (str));
+	    }
+	  else
+	    {
+	      // division
+	      current_column++;
+	      return Token::make (DIV, loc);
+	    }
+	case '%':
+	  if (peek_input () == '=')
+	    {
+	      // modulo-assign
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (PERCENT_EQ, loc);
+	    }
+	  else
+	    {
+	      // modulo
+	      current_column++;
+	      return Token::make (PERCENT, loc);
+	    }
+	case '^':
+	  if (peek_input () == '=')
+	    {
+	      // xor-assign?
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (CARET_EQ, loc);
+	    }
+	  else
+	    {
+	      // xor?
+	      current_column++;
+	      return Token::make (CARET, loc);
+	    }
+	case '<':
+	  if (peek_input () == '<')
+	    {
+	      if (peek_input (1) == '=')
+		{
+		  // left-shift assign
+		  skip_input (1);
+		  current_column += 3;
+
+		  return Token::make (LEFT_SHIFT_EQ, loc);
+		}
+	      else
+		{
+		  // left-shift
+		  skip_input ();
+		  current_column += 2;
+
+		  return Token::make (LEFT_SHIFT, loc);
+		}
+	    }
+	  else if (peek_input () == '=')
+	    {
+	      // smaller than or equal to
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (LESS_OR_EQUAL, loc);
+	    }
+	  else
+	    {
+	      // smaller than
+	      current_column++;
+	      return Token::make (LEFT_ANGLE, loc);
+	    }
+	  break;
+	case '>':
+	  if (peek_input () == '>')
+	    {
+	      if (peek_input (1) == '=')
+		{
+		  // right-shift-assign
+		  skip_input (1);
+		  current_column += 3;
+
+		  return Token::make (RIGHT_SHIFT_EQ, loc);
+		}
+	      else
+		{
+		  // right-shift
+		  skip_input ();
+		  current_column += 2;
+
+		  return Token::make (RIGHT_SHIFT, loc);
+		}
+	    }
+	  else if (peek_input () == '=')
+	    {
+	      // larger than or equal to
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (GREATER_OR_EQUAL, loc);
+	    }
+	  else
+	    {
+	      // larger than
+	      current_column++;
+	      return Token::make (RIGHT_ANGLE, loc);
+	    }
+	case ':':
+	  if (peek_input () == ':')
+	    {
+	      // scope resolution ::
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (SCOPE_RESOLUTION, loc);
+	    }
+	  else
+	    {
+	      // single colon :
+	      current_column++;
+	      return Token::make (COLON, loc);
+	    }
+	case '!':
+	  // no special handling for macros in lexer?
+	  if (peek_input () == '=')
+	    {
+	      // not equal boolean operator
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (NOT_EQUAL, loc);
+	    }
+	  else
+	    {
+	      // not equal unary operator
+	      current_column++;
+
+	      return Token::make (EXCLAM, loc);
+	    }
+	case '?':
+	  current_column++;
+	  return Token::make (QUESTION_MARK, loc);
+	case '#':
+	  current_column++;
+	  return Token::make (HASH, loc);
+	case '[':
+	  current_column++;
+	  return Token::make (LEFT_SQUARE, loc);
+	case ']':
+	  current_column++;
+	  return Token::make (RIGHT_SQUARE, loc);
+	case '{':
+	  current_column++;
+	  return Token::make (LEFT_CURLY, loc);
+	case '}':
+	  current_column++;
+	  return Token::make (RIGHT_CURLY, loc);
+	case '@':
+	  current_column++;
+	  return Token::make (PATTERN_BIND, loc);
+	case '$':
+	  current_column++;
+	  return Token::make (DOLLAR_SIGN, loc);
+	case '~':
+	  current_column++;
+	  return Token::make (TILDE, loc);
+	case '\\':
+	  current_column++;
+	  return Token::make (BACKSLASH, loc);
+	case '`':
+	  current_column++;
+	  return Token::make (BACKTICK, loc);
+	case '|':
+	  if (peek_input () == '=')
+	    {
+	      // bitwise or-assign?
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (PIPE_EQ, loc);
+	    }
+	  else if (peek_input () == '|')
+	    {
+	      // logical or
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (OR, loc);
+	    }
+	  else
+	    {
+	      // bitwise or
+	      current_column++;
+
+	      return Token::make (PIPE, loc);
+	    }
+	case '&':
+	  if (peek_input () == '=')
+	    {
+	      // bitwise and-assign?
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (AMP_EQ, loc);
+	    }
+	  else if (peek_input () == '&')
+	    {
+	      // logical and
+	      skip_input ();
+	      current_column += 2;
+
+	      return Token::make (LOGICAL_AND, loc);
+	    }
+	  else
+	    {
+	      // bitwise and/reference
+	      current_column++;
+
+	      return Token::make (AMP, loc);
+	    }
+	case '.':
+	  if (peek_input () == '.')
+	    {
+	      if (peek_input (1) == '.')
+		{
+		  // ellipsis
+		  skip_input (1);
+		  current_column += 3;
+
+		  return Token::make (ELLIPSIS, loc);
+		}
+	      else if (peek_input (1) == '=')
+		{
+		  // ..=
+		  skip_input (1);
+		  current_column += 3;
+
+		  return Token::make (DOT_DOT_EQ, loc);
+		}
+	      else
+		{
+		  // ..
+		  skip_input ();
+		  current_column += 2;
+
+		  return Token::make (DOT_DOT, loc);
+		}
+	    }
+	  else /*if (!ISDIGIT (peek_input ()))*/
+	    {
+	      // single dot .
+	      // Only if followed by a non-number - otherwise is float
+	      // nope, float cannot start with '.'.
+	      current_column++;
+	      return Token::make (DOT, loc);
+	    }
+	}
+      // TODO: special handling of _ in the lexer? instead of being identifier
+
+      // byte character, byte string and raw byte string literals
+      if (current_char == 'b')
+	{
+	  if (peek_input () == '\'')
+	    return parse_byte_char (loc);
+	  else if (peek_input () == '"')
+	    return parse_byte_string (loc);
+	  else if (peek_input () == 'r'
+		   && (peek_input (1) == '#' || peek_input (1) == '"'))
+	    return parse_raw_byte_string (loc);
+	}
+
+      // raw identifiers and raw strings
+      if (current_char == 'r')
+	{
+	  int peek = peek_input ();
+	  int peek1 = peek_input (1);
+
+	  if (peek == '#' && (ISALPHA (peek1) || peek1 == '_'))
+	    {
+	      TokenPtr raw_ident_ptr = parse_raw_identifier (loc);
+	      if (raw_ident_ptr != nullptr)
+		return raw_ident_ptr;
+	      else
+		continue; /* input got parsed, it just wasn't valid. An error
+			     was produced. */
+	    }
+	  else
+	    {
+	      TokenPtr maybe_raw_string_ptr = maybe_parse_raw_string (loc);
+	      if (maybe_raw_string_ptr != nullptr)
+		return maybe_raw_string_ptr;
+	    }
+	}
+
+      // find identifiers and keywords
+      if (ISALPHA (current_char) || current_char == '_')
+	return parse_identifier_or_keyword (loc);
+
+      // int and float literals
+      if (ISDIGIT (current_char))
+	{ //  _ not allowed as first char
+	  if (current_char == '0'
+	      && is_non_decimal_int_literal_separator (peek_input ()))
+	    {
+	      // handle binary, octal, hex literals
+	      TokenPtr non_dec_int_lit_ptr
+		= parse_non_decimal_int_literals (loc);
+	      if (non_dec_int_lit_ptr != nullptr)
+		return non_dec_int_lit_ptr;
+	    }
+	  else
+	    {
+	      // handle decimals (integer or float)
+	      TokenPtr decimal_or_float_ptr = parse_decimal_int_or_float (loc);
+	      if (decimal_or_float_ptr != nullptr)
+		return decimal_or_float_ptr;
+	    }
+	}
+
+      // string literals
+      if (current_char == '"')
+	return parse_string (loc);
+
+      // char literals and lifetime names
+      if (current_char == '\'')
+	{
+	  TokenPtr char_or_lifetime_ptr = parse_char_or_lifetime (loc);
+	  if (char_or_lifetime_ptr != nullptr)
+	    return char_or_lifetime_ptr;
+	}
+
+      // DEBUG: check for specific character problems:
+      if (current_char == '0')
+	rust_debug ("'0' uncaught before unexpected character");
+      else if (current_char == ']')
+	rust_debug ("']' uncaught before unexpected character");
+      else if (current_char == 0x5d)
+	rust_debug ("whatever 0x5d is (not '0' or ']') uncaught before "
+		    "unexpected character");
+
+      // didn't match anything so error
+      rust_error_at (loc, "unexpected character %<%x%>", current_char);
+      current_column++;
+    }
+}
+
+// Parses in a type suffix.
+std::pair<PrimitiveCoreType, int>
+Lexer::parse_in_type_suffix ()
+{
+  std::string suffix;
+  suffix.reserve (5);
+
+  int additional_length_offset = 0;
+
+  // get suffix
+  while (ISALPHA (current_char) || ISDIGIT (current_char)
+	 || current_char == '_')
+    {
+      if (current_char == '_')
+	{
+	  // don't add _ to suffix
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  additional_length_offset++;
+
+	  continue;
+	}
+
+      additional_length_offset++;
+
+      suffix += current_char;
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  if (suffix.empty ())
+    {
+      // no type suffix: do nothing but also no error
+      return std::make_pair (CORETYPE_UNKNOWN, additional_length_offset);
+    }
+  else if (suffix == "f32")
+    {
+      return std::make_pair (CORETYPE_F32, additional_length_offset);
+    }
+  else if (suffix == "f64")
+    {
+      return std::make_pair (CORETYPE_F64, additional_length_offset);
+    }
+  else if (suffix == "i8")
+    {
+      return std::make_pair (CORETYPE_I8, additional_length_offset);
+    }
+  else if (suffix == "i16")
+    {
+      return std::make_pair (CORETYPE_I16, additional_length_offset);
+    }
+  else if (suffix == "i32")
+    {
+      return std::make_pair (CORETYPE_I32, additional_length_offset);
+    }
+  else if (suffix == "i64")
+    {
+      return std::make_pair (CORETYPE_I64, additional_length_offset);
+    }
+  else if (suffix == "i128")
+    {
+      return std::make_pair (CORETYPE_I128, additional_length_offset);
+    }
+  else if (suffix == "isize")
+    {
+      return std::make_pair (CORETYPE_ISIZE, additional_length_offset);
+    }
+  else if (suffix == "u8")
+    {
+      return std::make_pair (CORETYPE_U8, additional_length_offset);
+    }
+  else if (suffix == "u16")
+    {
+      return std::make_pair (CORETYPE_U16, additional_length_offset);
+    }
+  else if (suffix == "u32")
+    {
+      return std::make_pair (CORETYPE_U32, additional_length_offset);
+    }
+  else if (suffix == "u64")
+    {
+      return std::make_pair (CORETYPE_U64, additional_length_offset);
+    }
+  else if (suffix == "u128")
+    {
+      return std::make_pair (CORETYPE_U128, additional_length_offset);
+    }
+  else if (suffix == "usize")
+    {
+      return std::make_pair (CORETYPE_USIZE, additional_length_offset);
+    }
+  else
+    {
+      rust_error_at (get_current_location (), "unknown number suffix %qs",
+		     suffix.c_str ());
+
+      return std::make_pair (CORETYPE_UNKNOWN, additional_length_offset);
+    }
+}
+
+// Parses in the exponent part (if any) of a float literal.
+std::pair<std::string, int>
+Lexer::parse_in_exponent_part ()
+{
+  int additional_length_offset = 0;
+  std::string str;
+  if (current_char == 'E' || current_char == 'e')
+    {
+      // add exponent to string as strtod works with it
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+
+      additional_length_offset++;
+
+      // special - and + handling
+      if (current_char == '-')
+	{
+	  str += '-';
+
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  additional_length_offset++;
+	}
+      else if (current_char == '+')
+	{
+	  // don't add + but still skip input
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  additional_length_offset++;
+	}
+
+      // parse another decimal number for exponent
+      auto str_length = parse_in_decimal ();
+      str += std::get<0> (str_length);
+      additional_length_offset += std::get<1> (str_length);
+    }
+  return std::make_pair (str, additional_length_offset);
+}
+
+// Parses a decimal integer.
+std::tuple<std::string, int, bool>
+Lexer::parse_in_decimal ()
+{
+  /* A pure decimal contains only digits.  */
+  bool pure_decimal = true;
+  int additional_length_offset = 0;
+  std::string str;
+  while (ISDIGIT (current_char) || current_char == '_')
+    {
+      if (current_char == '_')
+	{
+	  pure_decimal = false;
+	  // don't add _ to number
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  additional_length_offset++;
+
+	  continue;
+	}
+
+      additional_length_offset++;
+
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+    }
+  return std::make_tuple (str, additional_length_offset, pure_decimal);
+}
+
+/* Parses escapes (and string continues) in "byte" strings and characters. Does
+ * not support unicode. */
+std::tuple<char, int, bool>
+Lexer::parse_escape (char opening_char)
+{
+  int additional_length_offset = 0;
+  char output_char = 0;
+
+  // skip to actual letter
+  skip_input ();
+  current_char = peek_input ();
+  additional_length_offset++;
+
+  switch (current_char)
+    {
+      case 'x': {
+	auto hex_escape_pair = parse_partial_hex_escape ();
+	long hexLong = hex_escape_pair.first;
+	additional_length_offset += hex_escape_pair.second;
+
+	if (hexLong > 255 || hexLong < 0)
+	  rust_error_at (
+	    get_current_location (),
+	    "byte \\x escape %<\\x%x%> out of range - allows up to %<\\xFF%>",
+	    static_cast<unsigned int> (hexLong));
+	/* TODO: restore capital for escape output - gcc pretty-printer doesn't
+	 * support %X directly */
+	char hexChar = static_cast<char> (hexLong);
+
+	output_char = hexChar;
+      }
+      break;
+    case 'n':
+      output_char = '\n';
+      break;
+    case 'r':
+      output_char = '\r';
+      break;
+    case 't':
+      output_char = '\t';
+      break;
+    case '\\':
+      output_char = '\\';
+      break;
+    case '0':
+      output_char = '\0';
+      break;
+    case '\'':
+      output_char = '\'';
+      break;
+    case '"':
+      output_char = '"';
+      break;
+    case 'u':
+      rust_error_at (get_current_location (),
+		     "cannot have a unicode escape \\u in a byte %s",
+		     opening_char == '\'' ? "character" : "string");
+      // Try to parse it anyway, just to skip it
+      parse_partial_unicode_escape ();
+      return std::make_tuple (output_char, additional_length_offset, false);
+    case '\r':
+    case '\n':
+      // string continue
+      return std::make_tuple (0, parse_partial_string_continue (), true);
+    default:
+      rust_error_at (get_current_location (),
+		     "unknown escape sequence %<\\%c%>", current_char);
+      // returns false if no parsing could be done
+      // return false;
+      return std::make_tuple (output_char, additional_length_offset, false);
+      break;
+    }
+  // all non-special cases (string continue) should skip their used char
+  skip_input ();
+  current_char = peek_input ();
+  additional_length_offset++;
+
+  // returns true if parsing was successful
+  // return true;
+  return std::make_tuple (output_char, additional_length_offset, false);
+}
+
+/* Parses an escape (or string continue) in a string or character. Supports
+ * unicode escapes. */
+std::tuple<Codepoint, int, bool>
+Lexer::parse_utf8_escape (char opening_char)
+{
+  Codepoint output_char;
+  int additional_length_offset = 0;
+
+  // skip to actual letter
+  skip_input ();
+  current_char = peek_input ();
+  additional_length_offset++;
+
+  switch (current_char)
+    {
+      case 'x': {
+	auto hex_escape_pair = parse_partial_hex_escape ();
+	long hexLong = hex_escape_pair.first;
+	additional_length_offset += hex_escape_pair.second;
+
+	if (hexLong > 127 || hexLong < 0)
+	  rust_error_at (
+	    get_current_location (),
+	    "ascii \\x escape %<\\x%x%> out of range - allows up to %<\\x7F%>",
+	    static_cast<unsigned int> (hexLong));
+	/* TODO: restore capital for escape output - gcc pretty-printer doesn't
+	 * support %X directly */
+	char hexChar = static_cast<char> (hexLong);
+
+	output_char = hexChar;
+      }
+      break;
+    case 'n':
+      output_char = '\n';
+      break;
+    case 'r':
+      output_char = '\r';
+      break;
+    case 't':
+      output_char = '\t';
+      break;
+    case '\\':
+      output_char = '\\';
+      break;
+    case '0':
+      output_char = '\0';
+      break;
+    case '\'':
+      output_char = '\'';
+      break;
+    case '"':
+      output_char = '"';
+      break;
+      case 'u': {
+	auto unicode_escape_pair = parse_partial_unicode_escape ();
+	output_char = unicode_escape_pair.first;
+	additional_length_offset += unicode_escape_pair.second;
+
+	return std::make_tuple (output_char, additional_length_offset, false);
+      }
+      break;
+    case '\r':
+    case '\n':
+      // string continue
+      return std::make_tuple (0, parse_partial_string_continue (), true);
+    default:
+      rust_error_at (get_current_location (),
+		     "unknown escape sequence %<\\%c%>", current_char);
+      // returns false if no parsing could be done
+      // return false;
+      return std::make_tuple (output_char, additional_length_offset, false);
+      break;
+    }
+  /* all non-special cases (unicode, string continue) should skip their used
+   * char */
+  skip_input ();
+  current_char = peek_input ();
+  additional_length_offset++;
+
+  // returns true if parsing was successful
+  // return true;
+  return std::make_tuple (output_char, additional_length_offset, false);
+}
+
+// Parses the body of a string continue that has been found in an escape.
+int
+Lexer::parse_partial_string_continue ()
+{
+  int additional_length_offset = 1;
+
+  // string continue
+  while (is_whitespace (current_char))
+    {
+      if (current_char == '\n')
+	{
+	  current_line++;
+	  current_column = 1;
+	  // tell line_table that new line starts
+	  start_line (current_line, max_column_hint);
+
+	  // reset "length"
+	  additional_length_offset = 1;
+
+	  // get next char
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  continue;
+	}
+
+      skip_input ();
+      current_char = peek_input ();
+      additional_length_offset++;
+    }
+
+  return additional_length_offset;
+}
+
+/* Parses the body of a '\x' escape. Note that it does not check that the number
+ * is valid and smaller than 255. */
+std::pair<long, int>
+Lexer::parse_partial_hex_escape ()
+{
+  // hex char string (null-terminated)
+  char hexNum[3] = {0, 0, 0};
+
+  // first hex char
+  current_char = peek_input (1);
+  int additional_length_offset = 1;
+
+  if (!is_x_digit (current_char))
+    {
+      rust_error_at (get_current_location (),
+		     "invalid character %<\\x%c%> in \\x sequence",
+		     current_char);
+      return std::make_pair (0, 0);
+    }
+  hexNum[0] = current_char;
+
+  // second hex char
+  skip_input ();
+  current_char = peek_input (1);
+  additional_length_offset++;
+
+  if (!is_x_digit (current_char))
+    {
+      rust_error_at (get_current_location (),
+		     "invalid character %<\\x%c%c%> in \\x sequence", hexNum[0],
+		     current_char);
+      return std::make_pair (0, 1);
+    }
+  skip_input ();
+  hexNum[1] = current_char;
+
+  long hexLong = std::strtol (hexNum, nullptr, 16);
+
+  return std::make_pair (hexLong, additional_length_offset);
+}
+
+// Parses the body of a unicode escape.
+std::pair<Codepoint, int>
+Lexer::parse_partial_unicode_escape ()
+{
+  skip_input ();
+  current_char = peek_input ();
+  int additional_length_offset = 0;
+
+  if (current_char != '{')
+    {
+      rust_error_at (get_current_location (),
+		     "unicode escape should start with %<{%>");
+      /* Skip what should probaby have been between brackets.  */
+      while (is_x_digit (current_char) || current_char == '_')
+	{
+	  skip_input ();
+	  current_char = peek_input ();
+	  additional_length_offset++;
+	}
+      return std::make_pair (Codepoint (0), additional_length_offset);
+    }
+
+  skip_input ();
+  current_char = peek_input ();
+  additional_length_offset++;
+
+  if (current_char == '_')
+    {
+      rust_error_at (get_current_location (),
+		     "unicode escape cannot start with %<_%>");
+      skip_input ();
+      current_char = peek_input ();
+      additional_length_offset++;
+      // fallthrough and try to parse the rest anyway
+    }
+
+  // parse unicode escape - 1-6 hex digits
+  std::string num_str;
+  num_str.reserve (6);
+
+  // loop through to add entire hex number to string
+  while (is_x_digit (current_char) || current_char == '_')
+    {
+      if (current_char == '_')
+	{
+	  // don't add _ to number
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  additional_length_offset++;
+
+	  continue;
+	}
+
+      additional_length_offset++;
+
+      // add raw hex numbers
+      num_str += current_char;
+
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  if (current_char == '}')
+    {
+      skip_input ();
+      current_char = peek_input ();
+      additional_length_offset++;
+    }
+  else
+    {
+      // actually an error, but allow propagation anyway Assume that
+      // wrong bracketm whitespace or single/double quotes are wrong
+      // termination, otherwise it is a wrong character, then skip to the actual
+      // terminator.
+      if (current_char == '{' || is_whitespace (current_char)
+	  || current_char == '\'' || current_char == '"')
+	{
+	  rust_error_at (get_current_location (),
+			 "expected terminating %<}%> in unicode escape");
+	  return std::make_pair (Codepoint (0), additional_length_offset);
+	}
+      else
+	{
+	  rust_error_at (get_current_location (),
+			 "invalid character %<%c%> in unicode escape",
+			 current_char);
+	  while (current_char != '}' && current_char != '{'
+		 && !is_whitespace (current_char) && current_char != '\''
+		 && current_char != '"')
+	    {
+	      skip_input ();
+	      current_char = peek_input ();
+	      additional_length_offset++;
+	    }
+	  // Consume the actual closing bracket if found
+	  if (current_char == '}')
+	    {
+	      skip_input ();
+	      current_char = peek_input ();
+	      additional_length_offset++;
+	    }
+	  return std::make_pair (Codepoint (0), additional_length_offset);
+	}
+    }
+
+  // ensure 1-6 hex characters
+  if (num_str.length () > 6 || num_str.length () < 1)
+    {
+      rust_error_at (get_current_location (),
+		     "unicode escape should be between 1 and 6 hex "
+		     "characters; it is %lu",
+		     (unsigned long) num_str.length ());
+      // return false;
+      return std::make_pair (Codepoint (0), additional_length_offset);
+    }
+
+  unsigned long hex_num = std::strtoul (num_str.c_str (), nullptr, 16);
+
+  if (hex_num > 0xd7ff && hex_num < 0xe000)
+    {
+      rust_error_at (
+	get_current_location (),
+	"unicode escape cannot be a surrogate value (D800 to DFFF)");
+      return std::make_pair (Codepoint (0), additional_length_offset);
+    }
+
+  if (hex_num > 0x10ffff)
+    {
+      rust_error_at (get_current_location (),
+		     "unicode escape cannot be larger than 10FFFF");
+      return std::make_pair (Codepoint (0), additional_length_offset);
+    }
+
+  // return true;
+  return std::make_pair (Codepoint (static_cast<uint32_t> (hex_num)),
+			 additional_length_offset);
+}
+
+// Parses a byte character.
+TokenPtr
+Lexer::parse_byte_char (Location loc)
+{
+  skip_input ();
+  current_column++;
+  // make current char the next character
+  current_char = peek_input ();
+
+  int length = 1;
+
+  // char to save
+  char byte_char = 0;
+
+  // detect escapes
+  if (current_char == '\\')
+    {
+      auto escape_length_pair = parse_escape ('\'');
+      byte_char = std::get<0> (escape_length_pair);
+      length += std::get<1> (escape_length_pair);
+
+      current_char = peek_input ();
+
+      if (current_char != '\'')
+	{
+	  rust_error_at (get_current_location (), "unclosed %<byte char%>");
+	}
+
+      skip_input ();
+      current_char = peek_input ();
+      length++; // go to next char
+    }
+  else if (current_char != '\'')
+    {
+      // otherwise, get character from direct input character
+      byte_char = current_char;
+
+      skip_input ();
+      current_char = peek_input ();
+      length++;
+
+      if (current_char != '\'')
+	{
+	  rust_error_at (get_current_location (), "unclosed %<byte char%>");
+	}
+
+      skip_input ();
+      current_char = peek_input ();
+      length++; // go to next char
+    }
+  else
+    {
+      rust_error_at (get_current_location (),
+		     "no character inside %<%> for %<byte char%>");
+    }
+
+  current_column += length;
+
+  return Token::make_byte_char (loc, byte_char);
+}
+
+// Parses a byte string.
+TokenPtr
+Lexer::parse_byte_string (Location loc)
+{
+  // byte string
+
+  // skip quote character
+  skip_input ();
+  current_column++;
+
+  std::string str;
+  str.reserve (16); // some sensible default
+
+  int length = 1;
+  current_char = peek_input ();
+
+  while (current_char != '"' && current_char != EOF)
+    {
+      if (current_char == '\\')
+	{
+	  auto escape_length_pair = parse_escape ('"');
+	  char output_char = std::get<0> (escape_length_pair);
+
+	  if (output_char == 0 && std::get<2> (escape_length_pair))
+	    length = std::get<1> (escape_length_pair) - 1;
+	  else
+	    length += std::get<1> (escape_length_pair);
+
+	  if (output_char != 0 || !std::get<2> (escape_length_pair))
+	    str += output_char;
+
+	  continue;
+	}
+
+      length++;
+
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  current_column += length;
+
+  if (current_char == '"')
+    {
+      current_column++;
+
+      skip_input ();
+      current_char = peek_input ();
+    }
+  else if (current_char == EOF)
+    {
+      rust_error_at (get_current_location (), "unended byte string literal");
+      return Token::make (END_OF_FILE, get_current_location ());
+    }
+  else
+    {
+      gcc_unreachable ();
+    }
+
+  str.shrink_to_fit ();
+
+  return Token::make_byte_string (loc, std::move (str));
+}
+
+// Parses a raw byte string.
+TokenPtr
+Lexer::parse_raw_byte_string (Location loc)
+{
+  // raw byte string literals
+  std::string str;
+  str.reserve (16); // some sensible default
+
+  int length = 1;
+  int hash_count = 0;
+
+  // get hash count at beginnning
+  skip_input ();
+  current_char = peek_input ();
+  length++;
+  while (current_char == '#')
+    {
+      hash_count++;
+      length++;
+
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  if (current_char != '"')
+    {
+      rust_error_at (get_current_location (),
+		     "raw byte string has no opening %<\"%>");
+    }
+
+  skip_input ();
+  current_char = peek_input ();
+  length++;
+
+  while (true)
+    {
+      if (current_char == '"')
+	{
+	  bool enough_hashes = true;
+
+	  for (int i = 0; i < hash_count; i++)
+	    {
+	      if (peek_input (i + 1) != '#')
+		{
+		  enough_hashes = false;
+		  break;
+		}
+	    }
+
+	  if (enough_hashes)
+	    {
+	      // skip enough input and peek enough input
+	      skip_input (hash_count);
+	      current_char = peek_input ();
+	      length += hash_count + 1;
+	      break;
+	    }
+	}
+
+      if ((unsigned char) current_char > 127)
+	{
+	  rust_error_at (get_current_location (),
+			 "character %<%c%> in raw byte string out of range",
+			 current_char);
+	  current_char = 0;
+	}
+
+      length++;
+
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  current_column += length;
+
+  str.shrink_to_fit ();
+
+  return Token::make_byte_string (loc, std::move (str));
+}
+
+// Parses a raw identifier.
+TokenPtr
+Lexer::parse_raw_identifier (Location loc)
+{
+  // raw identifier
+  std::string str;
+  str.reserve (16); // default
+
+  skip_input ();
+  current_char = peek_input ();
+
+  current_column += 2;
+
+  bool first_is_underscore = current_char == '_';
+
+  int length = 0;
+  current_char = peek_input ();
+  // loop through entire name
+  while (ISALPHA (current_char) || ISDIGIT (current_char)
+	 || current_char == '_')
+    {
+      length++;
+
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  current_column += length;
+
+  // if just a single underscore, not an identifier
+  if (first_is_underscore && length == 1)
+    rust_error_at (get_current_location (),
+		   "%<_%> is not a valid raw identifier");
+
+  if (str == "crate" || str == "extern" || str == "self" || str == "super"
+      || str == "Self")
+    {
+      rust_error_at (get_current_location (),
+		     "%qs is a forbidden raw identifier", str.c_str ());
+
+      return nullptr;
+    }
+  else
+    {
+      str.shrink_to_fit ();
+
+      return Token::make_identifier (loc, std::move (str));
+    }
+}
+
+// skip broken string input (unterminated strings)
+void
+Lexer::skip_broken_string_input (int current_char)
+{
+  while (current_char != '"' && current_char != EOF)
+    {
+      if (current_char == '\n')
+	{
+	  current_line++;
+	  current_column = 1;
+	}
+      else
+	{
+	  current_column++;
+	}
+      skip_input ();
+      current_char = peek_input ();
+    }
+  if (current_char == '"')
+    {
+      current_column++;
+
+      skip_input ();
+      current_char = peek_input ();
+    }
+  rust_debug ("skipped to %d:%d due to bad quotes", current_line,
+	      current_column);
+}
+
+// Parses a unicode string.
+TokenPtr
+Lexer::parse_string (Location loc)
+{
+  Codepoint current_char32;
+
+  std::string str;
+  str.reserve (16); // some sensible default
+
+  int length = 1;
+  current_char32 = peek_codepoint_input ();
+
+  // FIXME: This fails if the input ends. How do we check for EOF?
+  while (current_char32.value != '"' && !current_char32.is_eof ())
+    {
+      if (current_char32.value == '\\')
+	{
+	  // parse escape
+	  auto utf8_escape_pair = parse_utf8_escape ('\'');
+	  current_char32 = std::get<0> (utf8_escape_pair);
+
+	  if (current_char32 == Codepoint (0) && std::get<2> (utf8_escape_pair))
+	    length = std::get<1> (utf8_escape_pair) - 1;
+	  else
+	    length += std::get<1> (utf8_escape_pair);
+
+	  if (current_char32 != Codepoint (0)
+	      || !std::get<2> (utf8_escape_pair))
+	    str += current_char32;
+
+	  // required as parsing utf8 escape only changes current_char
+	  current_char32 = peek_codepoint_input ();
+
+	  continue;
+	}
+
+      length += get_input_codepoint_length ();
+
+      str += current_char32;
+      skip_codepoint_input ();
+      current_char32 = peek_codepoint_input ();
+    }
+
+  current_column += length;
+
+  if (current_char32.value == '"')
+    {
+      current_column++;
+
+      skip_input ();
+      current_char = peek_input ();
+    }
+  else if (current_char32.is_eof ())
+    {
+      rust_error_at (get_current_location (), "unended string literal");
+      return Token::make (END_OF_FILE, get_current_location ());
+    }
+  else
+    {
+      gcc_unreachable ();
+    }
+
+  str.shrink_to_fit ();
+  return Token::make_string (loc, std::move (str));
+}
+
+// Parses an identifier or keyword.
+TokenPtr
+Lexer::parse_identifier_or_keyword (Location loc)
+{
+  std::string str;
+  str.reserve (16); // default
+  str += current_char;
+
+  bool first_is_underscore = current_char == '_';
+
+  int length = 1;
+  current_char = peek_input ();
+  // loop through entire name
+  while (ISALPHA (current_char) || ISDIGIT (current_char)
+	 || current_char == '_')
+    {
+      length++;
+
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  current_column += length;
+
+  // if just a single underscore, not an identifier
+  if (first_is_underscore && length == 1)
+    return Token::make (UNDERSCORE, loc);
+
+  str.shrink_to_fit ();
+
+  TokenId keyword = classify_keyword (str);
+  if (keyword == IDENTIFIER)
+    return Token::make_identifier (loc, std::move (str));
+  else
+    return Token::make (keyword, loc);
+}
+
+// Possibly returns a raw string token if it exists - otherwise returns null.
+TokenPtr
+Lexer::maybe_parse_raw_string (Location loc)
+{
+  int peek_index = 0;
+  while (peek_input (peek_index) == '#')
+    peek_index++;
+
+  if (peek_input (peek_index) == '"')
+    return parse_raw_string (loc, peek_index);
+  else
+    return nullptr;
+}
+
+// Returns a raw string token.
+TokenPtr
+Lexer::parse_raw_string (Location loc, int initial_hash_count)
+{
+  // raw string literals
+  std::string str;
+  str.reserve (16); // some sensible default
+
+  int length = 1 + initial_hash_count;
+
+  if (initial_hash_count > 0)
+    skip_input (initial_hash_count - 1);
+
+  current_char = peek_input ();
+
+  if (current_char != '"')
+    rust_error_at (get_current_location (), "raw string has no opening %<\"%>");
+
+  length++;
+  skip_input ();
+  Codepoint current_char32 = peek_codepoint_input ();
+
+  while (!current_char32.is_eof ())
+    {
+      if (current_char32.value == '"')
+	{
+	  bool enough_hashes = true;
+
+	  for (int i = 0; i < initial_hash_count; i++)
+	    {
+	      if (peek_input (i + 1) != '#')
+		{
+		  enough_hashes = false;
+		  break;
+		}
+	    }
+
+	  if (enough_hashes)
+	    {
+	      // skip enough input and peek enough input
+	      skip_input (initial_hash_count);
+	      current_char = peek_input ();
+	      length += initial_hash_count + 1;
+	      break;
+	    }
+	}
+
+      length++;
+
+      str += current_char32;
+      skip_codepoint_input ();
+      current_char32 = peek_codepoint_input ();
+    }
+
+  current_column += length;
+
+  str.shrink_to_fit ();
+
+  return Token::make_string (loc, std::move (str));
+}
+
+template <typename IsDigitFunc>
+TokenPtr
+Lexer::parse_non_decimal_int_literal (Location loc, IsDigitFunc is_digit_func,
+				      std::string existent_str, int base)
+{
+  int length = 1;
+
+  skip_input ();
+  current_char = peek_input ();
+
+  length++;
+
+  // loop through to add entire number to string
+  while (is_digit_func (current_char) || current_char == '_')
+    {
+      if (current_char == '_')
+	{
+	  // don't add _ to number
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  length++;
+
+	  continue;
+	}
+
+      length++;
+
+      // add raw numbers
+      existent_str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+    }
+
+  // convert value to decimal representation
+  long dec_num = std::strtol (existent_str.c_str (), nullptr, base);
+
+  existent_str = std::to_string (dec_num);
+
+  // parse in type suffix if it exists
+  auto type_suffix_pair = parse_in_type_suffix ();
+  PrimitiveCoreType type_hint = type_suffix_pair.first;
+  length += type_suffix_pair.second;
+
+  current_column += length;
+
+  if (type_hint == CORETYPE_F32 || type_hint == CORETYPE_F64)
+    {
+      rust_error_at (get_current_location (),
+		     "invalid type suffix %qs for integer (%s) literal",
+		     get_type_hint_string (type_hint),
+		     base == 16
+		       ? "hex"
+		       : (base == 8 ? "octal"
+				    : (base == 2 ? "binary"
+						 : "<insert unknown base>")));
+      return nullptr;
+    }
+  return Token::make_int (loc, std::move (existent_str), type_hint);
+}
+
+// Parses a hex, binary or octal int literal.
+TokenPtr
+Lexer::parse_non_decimal_int_literals (Location loc)
+{
+  std::string str;
+  str.reserve (16); // some sensible default
+  str += current_char;
+
+  current_char = peek_input ();
+
+  if (current_char == 'x')
+    {
+      // hex (integer only)
+      return parse_non_decimal_int_literal (loc, is_x_digit, str + "x", 16);
+    }
+  else if (current_char == 'o')
+    {
+      // octal (integer only)
+      return parse_non_decimal_int_literal (loc, is_octal_digit,
+					    std::move (str), 8);
+    }
+  else if (current_char == 'b')
+    {
+      // binary (integer only)
+      return parse_non_decimal_int_literal (loc, is_bin_digit, std::move (str),
+					    2);
+    }
+  else
+    {
+      return nullptr;
+    }
+}
+
+// Parses a decimal-based int literal or float literal.
+TokenPtr
+Lexer::parse_decimal_int_or_float (Location loc)
+{
+  std::string str;
+  str.reserve (16); // some sensible default
+  str += current_char;
+
+  int length = 1;
+  bool first_zero = current_char == '0';
+
+  current_char = peek_input ();
+
+  // parse initial decimal integer (or first integer part of float) literal
+  auto initial_decimal = parse_in_decimal ();
+  str += std::get<0> (initial_decimal);
+  length += std::get<1> (initial_decimal);
+
+  // detect float literal
+  if (current_char == '.' && is_float_digit (peek_input (1)))
+    {
+      // float with a '.', parse another decimal into it
+
+      // add . to str
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+      length++;
+
+      // parse another decimal number for float
+      auto second_decimal = parse_in_decimal ();
+      str += std::get<0> (second_decimal);
+      length += std::get<1> (second_decimal);
+
+      // parse in exponent part if it exists
+      auto exponent_pair = parse_in_exponent_part ();
+      str += exponent_pair.first;
+      length += exponent_pair.second;
+
+      // parse in type suffix if it exists
+      auto type_suffix_pair = parse_in_type_suffix ();
+      PrimitiveCoreType type_hint = type_suffix_pair.first;
+      length += type_suffix_pair.second;
+
+      if (type_hint != CORETYPE_F32 && type_hint != CORETYPE_F64
+	  && type_hint != CORETYPE_UNKNOWN)
+	{
+	  rust_error_at (get_current_location (),
+			 "invalid type suffix %qs for floating-point literal",
+			 get_type_hint_string (type_hint));
+	  // ignore invalid type suffix as everything else seems fine
+	  type_hint = CORETYPE_UNKNOWN;
+	}
+
+      current_column += length;
+
+      str.shrink_to_fit ();
+      return Token::make_float (loc, std::move (str), type_hint);
+    }
+  else if (current_char == '.' && check_valid_float_dot_end (peek_input (1)))
+    {
+      // float that is just an integer with a terminating '.' character
+
+      // add . to str
+      str += current_char;
+      skip_input ();
+      current_char = peek_input ();
+      length++;
+
+      // add a '0' after the . to prevent ambiguity
+      str += '0';
+
+      // type hint not allowed
+
+      current_column += length;
+
+      str.shrink_to_fit ();
+      return Token::make_float (loc, std::move (str), CORETYPE_UNKNOWN);
+    }
+  else if (current_char == 'E' || current_char == 'e')
+    {
+      // exponent float with no '.' character
+
+      // parse exponent part
+      auto exponent_pair = parse_in_exponent_part ();
+      str += exponent_pair.first;
+      length += exponent_pair.second;
+
+      // parse in type suffix if it exists
+      auto type_suffix_pair = parse_in_type_suffix ();
+      PrimitiveCoreType type_hint = type_suffix_pair.first;
+      length += type_suffix_pair.second;
+
+      if (type_hint != CORETYPE_F32 && type_hint != CORETYPE_F64
+	  && type_hint != CORETYPE_UNKNOWN)
+	{
+	  rust_error_at (get_current_location (),
+			 "invalid type suffix %qs for floating-point literal",
+			 get_type_hint_string (type_hint));
+	  // ignore invalid type suffix as everything else seems fine
+	  type_hint = CORETYPE_UNKNOWN;
+	}
+
+      current_column += length;
+
+      str.shrink_to_fit ();
+      return Token::make_float (loc, std::move (str), type_hint);
+    }
+  else
+    {
+      // is an integer
+
+      // parse in type suffix if it exists
+      auto type_suffix_pair = parse_in_type_suffix ();
+      PrimitiveCoreType type_hint = type_suffix_pair.first;
+      /* A "real" pure decimal doesn't have a suffix and no zero prefix.  */
+      if (type_hint == CORETYPE_UNKNOWN)
+	{
+	  bool pure_decimal = std::get<2> (initial_decimal);
+	  if (pure_decimal && (!first_zero || str.size () == 1))
+	    type_hint = CORETYPE_PURE_DECIMAL;
+	}
+      length += type_suffix_pair.second;
+
+      current_column += length;
+
+      str.shrink_to_fit ();
+      return Token::make_int (loc, std::move (str), type_hint);
+    }
+}
+
+TokenPtr
+Lexer::parse_char_or_lifetime (Location loc)
+{
+  Codepoint current_char32;
+
+  int length = 1;
+
+  current_char32 = peek_codepoint_input ();
+  if (current_char32.is_eof ())
+    return nullptr;
+
+  // parse escaped char literal
+  if (current_char32.value == '\\')
+    {
+      // parse escape
+      auto utf8_escape_pair = parse_utf8_escape ('\'');
+      current_char32 = std::get<0> (utf8_escape_pair);
+      length += std::get<1> (utf8_escape_pair);
+
+      if (peek_codepoint_input ().value != '\'')
+	{
+	  rust_error_at (get_current_location (), "unended character literal");
+	}
+      else
+	{
+	  skip_codepoint_input ();
+	  current_char = peek_input ();
+	  length++;
+	}
+
+      current_column += length;
+
+      return Token::make_char (loc, current_char32);
+    }
+  else
+    {
+      skip_codepoint_input ();
+
+      if (peek_codepoint_input ().value == '\'')
+	{
+	  // parse non-escaped char literal
+
+	  // skip the ' character
+	  skip_input ();
+	  current_char = peek_input ();
+
+	  // TODO fix due to different widths of utf-8 chars?
+	  current_column += 3;
+
+	  return Token::make_char (loc, current_char32);
+	}
+      else if (ISDIGIT (current_char32.value) || ISALPHA (current_char32.value)
+	       || current_char32.value == '_')
+	{
+	  // parse lifetime name
+	  std::string str;
+	  str += current_char32;
+	  length++;
+
+	  current_char = peek_input ();
+	  while (ISDIGIT (current_char) || ISALPHA (current_char)
+		 || current_char == '_')
+	    {
+	      str += current_char;
+	      skip_input ();
+	      current_char = peek_input ();
+	      length++;
+	    }
+
+	  current_column += length;
+
+	  str.shrink_to_fit ();
+	  return Token::make_lifetime (loc, std::move (str));
+	}
+      else
+	{
+	  rust_error_at (
+	    get_current_location (),
+	    "expected %' after character constant in character literal");
+	  return nullptr;
+	}
+    }
+}
+
+// Returns the length of the codepoint at the current position.
+int
+Lexer::get_input_codepoint_length ()
+{
+  uint8_t input = peek_input ();
+
+  if ((int8_t) input == EOF)
+    return 0;
+
+  if (input < 128)
+    {
+      // ascii -- 1 byte
+      // return input;
+
+      return 1;
+    }
+  else if ((input & 0xC0) == 0x80)
+    {
+      // invalid (continuation; can't be first char)
+      // return 0xFFFE;
+
+      return 0;
+    }
+  else if ((input & 0xE0) == 0xC0)
+    {
+      // 2 bytes
+      uint8_t input2 = peek_input (1);
+      if ((input2 & 0xC0) != 0x80)
+	return 0;
+      // return 0xFFFE;
+
+      // uint32_t output = ((input & 0x1F) << 6) | ((input2 & 0x3F) << 0);
+      // return output;
+      return 2;
+    }
+  else if ((input & 0xF0) == 0xE0)
+    {
+      // 3 bytes
+      uint8_t input2 = peek_input (1);
+      if ((input2 & 0xC0) != 0x80)
+	return 0;
+      // return 0xFFFE;
+
+      uint8_t input3 = peek_input (2);
+      if ((input3 & 0xC0) != 0x80)
+	return 0;
+      // return 0xFFFE;
+
+      /*uint32_t output
+	= ((input & 0x0F) << 12) | ((input2 & 0x3F) << 6) | ((input3 & 0x3F) <<
+      0); return output;*/
+      return 3;
+    }
+  else if ((input & 0xF8) == 0xF0)
+    {
+      // 4 bytes
+      uint8_t input2 = peek_input (1);
+      if ((input2 & 0xC0) != 0x80)
+	return 0;
+      // return 0xFFFE;
+
+      uint8_t input3 = peek_input (2);
+      if ((input3 & 0xC0) != 0x80)
+	return 0;
+      // return 0xFFFE;
+
+      uint8_t input4 = peek_input (3);
+      if ((input4 & 0xC0) != 0x80)
+	return 0;
+      // return 0xFFFE;
+
+      /*uint32_t output = ((input & 0x07) << 18) | ((input2 & 0x3F) << 12)
+			| ((input3 & 0x3F) << 6) | ((input4 & 0x3F) << 0);
+      return output;*/
+      return 4;
+    }
+  else
+    {
+      rust_error_at (get_current_location (),
+		     "invalid UTF-8 [FIRST] (too long)");
+      return 0;
+    }
+}
+
+// Returns the codepoint at the current position.
+Codepoint
+Lexer::peek_codepoint_input ()
+{
+  uint8_t input = peek_input ();
+
+  if ((int8_t) input == EOF)
+    return Codepoint::eof ();
+
+  if (input < 128)
+    {
+      // ascii -- 1 byte
+      return {input};
+    }
+  else if ((input & 0xC0) == 0x80)
+    {
+      // invalid (continuation; can't be first char)
+      return {0xFFFE};
+    }
+  else if ((input & 0xE0) == 0xC0)
+    {
+      // 2 bytes
+      uint8_t input2 = peek_input (1);
+      if ((input2 & 0xC0) != 0x80)
+	return {0xFFFE};
+
+      uint32_t output = ((input & 0x1F) << 6) | ((input2 & 0x3F) << 0);
+      return {output};
+    }
+  else if ((input & 0xF0) == 0xE0)
+    {
+      // 3 bytes
+      uint8_t input2 = peek_input (1);
+      if ((input2 & 0xC0) != 0x80)
+	return {0xFFFE};
+
+      uint8_t input3 = peek_input (2);
+      if ((input3 & 0xC0) != 0x80)
+	return {0xFFFE};
+
+      uint32_t output = ((input & 0x0F) << 12) | ((input2 & 0x3F) << 6)
+			| ((input3 & 0x3F) << 0);
+      return {output};
+    }
+  else if ((input & 0xF8) == 0xF0)
+    {
+      // 4 bytes
+      uint8_t input2 = peek_input (1);
+      if ((input2 & 0xC0) != 0x80)
+	return {0xFFFE};
+
+      uint8_t input3 = peek_input (2);
+      if ((input3 & 0xC0) != 0x80)
+	return {0xFFFE};
+
+      uint8_t input4 = peek_input (3);
+      if ((input4 & 0xC0) != 0x80)
+	return {0xFFFE};
+
+      uint32_t output = ((input & 0x07) << 18) | ((input2 & 0x3F) << 12)
+			| ((input3 & 0x3F) << 6) | ((input4 & 0x3F) << 0);
+      return {output};
+    }
+  else
+    {
+      rust_error_at (get_current_location (),
+		     "invalid UTF-8 [SECND] (too long)");
+      return {0xFFFE};
+    }
+}
+
+void
+Lexer::skip_codepoint_input ()
+{
+  int toSkip = get_input_codepoint_length ();
+  gcc_assert (toSkip >= 1);
+
+  skip_input (toSkip - 1);
+}
+
+int
+Lexer::test_get_input_codepoint_n_length (int n_start_offset)
+{
+  uint8_t input = peek_input (n_start_offset);
+
+  if (input < 128)
+    {
+      // ascii -- 1 byte
+      // return input;
+      return 1;
+    }
+  else if ((input & 0xC0) == 0x80)
+    {
+      // invalid (continuation; can't be first char)
+      // return 0xFFFE;
+      return 0;
+    }
+  else if ((input & 0xE0) == 0xC0)
+    {
+      // 2 bytes
+      uint8_t input2 = peek_input (n_start_offset + 1);
+      if ((input2 & 0xC0) != 0x80)
+	// return 0xFFFE;
+	return 0;
+
+      // uint32_t output = ((input & 0x1F) << 6) | ((input2 & 0x3F) << 0);
+      // return output;
+      return 2;
+    }
+  else if ((input & 0xF0) == 0xE0)
+    {
+      // 3 bytes
+      uint8_t input2 = peek_input (n_start_offset + 1);
+      if ((input2 & 0xC0) != 0x80)
+	// return 0xFFFE;
+	return 0;
+
+      uint8_t input3 = peek_input (n_start_offset + 2);
+      if ((input3 & 0xC0) != 0x80)
+	// return 0xFFFE;
+	return 0;
+
+      /*uint32_t output
+	= ((input & 0x0F) << 12) | ((input2 & 0x3F) << 6) | ((input3 & 0x3F) <<
+      0); return output;*/
+      return 3;
+    }
+  else if ((input & 0xF8) == 0xF0)
+    {
+      // 4 bytes
+      uint8_t input2 = peek_input (n_start_offset + 1);
+      if ((input2 & 0xC0) != 0x80)
+	// return 0xFFFE;
+	return 0;
+
+      uint8_t input3 = peek_input (n_start_offset + 2);
+      if ((input3 & 0xC0) != 0x80)
+	// return 0xFFFE;
+	return 0;
+
+      uint8_t input4 = peek_input (n_start_offset + 3);
+      if ((input4 & 0xC0) != 0x80)
+	// return 0xFFFE;
+	return 0;
+
+      /*uint32_t output = ((input & 0x07) << 18) | ((input2 & 0x3F) << 12)
+			| ((input3 & 0x3F) << 6) | ((input4 & 0x3F) << 0);
+      return output;*/
+      return 4;
+    }
+  else
+    {
+      rust_error_at (get_current_location (),
+		     "invalid UTF-8 [THIRD] (too long)");
+      return 0;
+    }
+}
+
+// peeks the codepoint input at n codepoints ahead of current codepoint - try
+// not to use
+Codepoint
+Lexer::test_peek_codepoint_input (int n)
+{
+  int totalOffset = 0;
+
+  // add up all offsets into total offset? does this do what I want?
+  for (int i = 0; i < n; i++)
+    {
+      totalOffset += test_get_input_codepoint_n_length (totalOffset);
+    }
+  // issues: this would have (at least) O(n) lookup time, not O(1) like the
+  // rest?
+
+  // TODO: implement if still needed
+
+  // error out of function as it is not implemented
+  gcc_assert (1 == 0);
+  return {0};
+  /*
+	  uint8_t input = peek_input();
+
+	  if (input < 128) {
+	      // ascii -- 1 byte
+	      return input;
+	  } else if ((input & 0xC0) == 0x80) {
+	      // invalid (continuation; can't be first char)
+	      return 0xFFFE;
+	  } else if ((input & 0xE0) == 0xC0) {
+	      // 2 bytes
+	      uint8_t input2 = peek_input(1);
+	      if ((input2 & 0xC0) != 0x80)
+		  return 0xFFFE;
+
+	      uint32_t output = ((input & 0x1F) << 6) | ((input2 & 0x3F) << 0);
+	      return output;
+	  } else if ((input & 0xF0) == 0xE0) {
+	      // 3 bytes
+	      uint8_t input2 = peek_input(1);
+	      if ((input2 & 0xC0) != 0x80)
+		  return 0xFFFE;
+
+	      uint8_t input3 = peek_input(2);
+	      if ((input3 & 0xC0) != 0x80)
+		  return 0xFFFE;
+
+	      uint32_t output
+		= ((input & 0x0F) << 12) | ((input2 & 0x3F) << 6) | ((input3 &
+     0x3F) << 0); return output; } else if ((input & 0xF8) == 0xF0) {
+	      // 4 bytes
+	      uint8_t input2 = peek_input(1);
+	      if ((input2 & 0xC0) != 0x80)
+		  return 0xFFFE;
+
+	      uint8_t input3 = peek_input(2);
+	      if ((input3 & 0xC0) != 0x80)
+		  return 0xFFFE;
+
+	      uint8_t input4 = peek_input(3);
+	      if ((input4 & 0xC0) != 0x80)
+		  return 0xFFFE;
+
+	      uint32_t output = ((input & 0x07) << 18) | ((input2 & 0x3F) << 12)
+				| ((input3 & 0x3F) << 6) | ((input4 & 0x3F) <<
+     0); return output; } else { rust_error_at(get_current_location(), "invalid
+     UTF-8 (too long)"); return 0xFFFE;
+	  }*/
+}
+
+void
+Lexer::split_current_token (TokenId new_left, TokenId new_right)
+{
+  /* TODO: assert that this TokenId is a "simple token" like punctuation and not
+   * like "IDENTIFIER"? */
+  Location current_loc = peek_token ()->get_locus ();
+  TokenPtr new_left_tok = Token::make (new_left, current_loc);
+  TokenPtr new_right_tok = Token::make (new_right, current_loc + 1);
+
+  token_queue.replace_current_value (std::move (new_left_tok));
+  token_queue.insert (1, std::move (new_right_tok));
+}
+
+void
+Lexer::start_line (int current_line, int current_column)
+{
+  if (line_map)
+    line_map->start_line (current_line, current_column);
+}
+
+} // namespace Rust
diff --git a/gcc/rust/lex/rust-lex.h b/gcc/rust/lex/rust-lex.h
new file mode 100644
index 00000000000..d5a6c53719f
--- /dev/null
+++ b/gcc/rust/lex/rust-lex.h
@@ -0,0 +1,271 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_LEX_H
+#define RUST_LEX_H
+
+#include "rust-linemap.h"
+#include "rust-buffered-queue.h"
+#include "rust-token.h"
+
+namespace Rust {
+// Simple wrapper for FILE* that simplifies destruction.
+struct RAIIFile
+{
+private:
+  FILE *file;
+  const char *filename;
+
+  void close ()
+  {
+    if (file != nullptr && file != stdin)
+      fclose (file);
+  }
+
+public:
+  RAIIFile (const char *filename) : filename (filename)
+  {
+    if (strcmp (filename, "-") == 0)
+      file = stdin;
+    else
+      file = fopen (filename, "r");
+  }
+
+  /**
+   * Create a RAIIFile from an existing instance of FILE*
+   */
+  RAIIFile (FILE *raw, const char *filename = nullptr)
+    : file (raw), filename (filename)
+  {}
+
+  RAIIFile (const RAIIFile &other) = delete;
+  RAIIFile &operator= (const RAIIFile &other) = delete;
+
+  // have to specify setting file to nullptr, otherwise unintended fclose occurs
+  RAIIFile (RAIIFile &&other) : file (other.file), filename (other.filename)
+  {
+    other.file = nullptr;
+  }
+
+  RAIIFile &operator= (RAIIFile &&other)
+  {
+    close ();
+    file = other.file;
+    filename = other.filename;
+    other.file = nullptr;
+
+    return *this;
+  }
+
+  static RAIIFile create_error () { return RAIIFile (nullptr, nullptr); }
+
+  ~RAIIFile () { close (); }
+
+  FILE *get_raw () { return file; }
+  const char *get_filename () { return filename; }
+
+  bool ok () const { return file; }
+};
+
+class Lexer
+{
+private:
+  // Request new Location for current column in line_table
+  Location get_current_location ();
+
+  // Skips the current input char.
+  void skip_input ();
+  // Advances current input char to n + 1 chars ahead of current position.
+  void skip_input (int n);
+
+  // Returns char n chars ahead of current position.
+  int peek_input ();
+  // Peeks the current char.
+  int peek_input (int n);
+
+  // Classifies keyword (i.e. gets id for keyword).
+  TokenId classify_keyword (const std::string &str);
+
+  // Builds a token from the input queue.
+  TokenPtr build_token ();
+
+  std::tuple<std::string, int, bool> parse_in_decimal ();
+  std::pair<std::string, int> parse_in_exponent_part ();
+  std::pair<PrimitiveCoreType, int> parse_in_type_suffix ();
+  std::tuple<char, int, bool> parse_escape (char opening_char);
+  std::tuple<Codepoint, int, bool> parse_utf8_escape (char opening_char);
+  int parse_partial_string_continue ();
+  std::pair<long, int> parse_partial_hex_escape ();
+  std::pair<Codepoint, int> parse_partial_unicode_escape ();
+
+  int get_input_codepoint_length ();
+  int test_get_input_codepoint_n_length (int n_start_offset);
+  Codepoint peek_codepoint_input ();
+  Codepoint test_peek_codepoint_input (int n);
+  void skip_codepoint_input ();
+  void skip_broken_string_input (int current_char);
+
+  TokenPtr parse_byte_char (Location loc);
+  TokenPtr parse_byte_string (Location loc);
+  TokenPtr parse_raw_byte_string (Location loc);
+  TokenPtr parse_raw_identifier (Location loc);
+  TokenPtr parse_string (Location loc);
+  TokenPtr maybe_parse_raw_string (Location loc);
+  TokenPtr parse_raw_string (Location loc, int initial_hash_count);
+  TokenPtr parse_non_decimal_int_literals (Location loc);
+  TokenPtr parse_decimal_int_or_float (Location loc);
+  TokenPtr parse_char_or_lifetime (Location loc);
+  TokenPtr parse_identifier_or_keyword (Location loc);
+
+  template <typename IsDigitFunc>
+  TokenPtr parse_non_decimal_int_literal (Location loc,
+					  IsDigitFunc is_digit_func,
+					  std::string existent_str, int base);
+
+public:
+  // Construct lexer with input file and filename provided
+  Lexer (const char *filename, RAIIFile input, Linemap *linemap);
+
+  // Lex the contents of a string instead of a file
+  Lexer (const std::string &input);
+
+  // dtor
+  ~Lexer ();
+
+  // don't allow copy semantics (for now, at least)
+  Lexer (const Lexer &other) = delete;
+  Lexer &operator= (const Lexer &other) = delete;
+
+  // enable move semantics
+  Lexer (Lexer &&other) = default;
+  Lexer &operator= (Lexer &&other) = default;
+
+  // Returns token n tokens ahead of current position.
+  const_TokenPtr peek_token (int n) { return token_queue.peek (n); }
+  // Peeks the current token.
+  const_TokenPtr peek_token () { return peek_token (0); }
+
+  // Advances current token to n + 1 tokens ahead of current position.
+  void skip_token (int n) { token_queue.skip (n); }
+  // Skips the current token.
+  void skip_token () { skip_token (0); }
+
+  // Replaces the current token with a specified token.
+  void replace_current_token (TokenPtr replacement);
+  // FIXME: don't use anymore
+
+  /* Splits the current token into two. Intended for use with nested generics
+   * closes (i.e. T<U<X>> where >> is wrongly lexed as one token). Note that
+   * this will only work with "simple" tokens like punctuation. */
+  void split_current_token (TokenId new_left, TokenId new_right);
+
+  Linemap *get_line_map () { return line_map; }
+  std::string get_filename () { return std::string (input.get_filename ()); }
+
+private:
+  void start_line (int current_line, int current_column);
+
+  // File for use as input.
+  RAIIFile input;
+  // TODO is this actually required? could just have file storage in InputSource
+
+  // Current line number.
+  int current_line;
+  // Current column number.
+  int current_column;
+  // Current character.
+  int current_char;
+  // Line map.
+  Linemap *line_map;
+
+  /* Max column number that can be quickly allocated - higher may require
+   * allocating new linemap */
+  static const int max_column_hint = 80;
+
+  // Input source wrapper thing.
+  class InputSource
+  {
+  public:
+    virtual ~InputSource () {}
+
+    // Overload operator () to return next char from input stream.
+    virtual int next () = 0;
+  };
+
+  class FileInputSource : public InputSource
+  {
+  private:
+    // Input source file.
+    FILE *input;
+
+  public:
+    // Create new input source from file.
+    FileInputSource (FILE *input) : input (input) {}
+
+    int next () override { return fgetc (input); }
+  };
+
+  class BufferInputSource : public InputSource
+  {
+  private:
+    const std::string &buffer;
+    size_t offs;
+
+  public:
+    // Create new input source from file.
+    BufferInputSource (const std::string &b, size_t offset)
+      : buffer (b), offs (offset)
+    {}
+
+    int next () override
+    {
+      if (offs >= buffer.size ())
+	return EOF;
+
+      return buffer.at (offs++);
+    }
+  };
+
+  // The input source for the lexer.
+  // InputSource input_source;
+  // Input file queue.
+  std::unique_ptr<InputSource> raw_input_source;
+  buffered_queue<int, InputSource &> input_queue;
+
+  // Token source wrapper thing.
+  struct TokenSource
+  {
+    // The lexer object that will use this TokenSource.
+    Lexer *lexer;
+
+    // Create a new TokenSource with given lexer.
+    TokenSource (Lexer *parLexer) : lexer (parLexer) {}
+
+    // Overload operator () to build token in lexer.
+    TokenPtr next () { return lexer->build_token (); }
+  };
+
+  // The token source for the lexer.
+  // TokenSource token_source;
+  // Token stream queue.
+  buffered_queue<std::shared_ptr<Token>, TokenSource> token_queue;
+};
+
+} // namespace Rust
+
+#endif
diff --git a/gcc/rust/lex/rust-token.cc b/gcc/rust/lex/rust-token.cc
new file mode 100644
index 00000000000..68313c20b1c
--- /dev/null
+++ b/gcc/rust/lex/rust-token.cc
@@ -0,0 +1,135 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-token.h"
+
+#include "rust-diagnostics.h" // for error_at
+
+namespace Rust {
+// Hackily defined way to get token description for enum value using x-macros
+const char *
+get_token_description (TokenId id)
+{
+  switch (id)
+    {
+#define RS_TOKEN(name, descr)                                                  \
+  case name:                                                                   \
+    return descr;
+#define RS_TOKEN_KEYWORD(x, y) RS_TOKEN (x, y)
+      RS_TOKEN_LIST
+#undef RS_TOKEN_KEYWORD
+#undef RS_TOKEN
+    default:
+      gcc_unreachable ();
+    }
+}
+
+/* Hackily defined way to get token description as a string for enum value using
+ * x-macros */
+const char *
+token_id_to_str (TokenId id)
+{
+  switch (id)
+    {
+#define RS_TOKEN(name, _)                                                      \
+  case name:                                                                   \
+    return #name;
+#define RS_TOKEN_KEYWORD(x, y) RS_TOKEN (x, y)
+      RS_TOKEN_LIST
+#undef RS_TOKEN_KEYWORD
+#undef RS_TOKEN
+    default:
+      gcc_unreachable ();
+    }
+}
+
+const char *
+get_type_hint_string (PrimitiveCoreType type)
+{
+  switch (type)
+    {
+    case CORETYPE_BOOL:
+      return "bool";
+    case CORETYPE_CHAR:
+      return "char";
+    case CORETYPE_STR:
+      return "str";
+    // case CORETYPE_INT:
+    case CORETYPE_ISIZE:
+      return "isize";
+    // case CORETYPE_UINT:
+    case CORETYPE_USIZE:
+      return "usize";
+    case CORETYPE_F32:
+      return "f32";
+    case CORETYPE_F64:
+      return "f64";
+    case CORETYPE_I8:
+      return "i8";
+    case CORETYPE_I16:
+      return "i16";
+    case CORETYPE_I32:
+      return "i32";
+    case CORETYPE_I64:
+      return "i64";
+    case CORETYPE_I128:
+      return "i128";
+    case CORETYPE_U8:
+      return "u8";
+    case CORETYPE_U16:
+      return "u16";
+    case CORETYPE_U32:
+      return "u32";
+    case CORETYPE_U64:
+      return "u64";
+    case CORETYPE_U128:
+      return "u128";
+    case CORETYPE_PURE_DECIMAL:
+      return "pure_decimal";
+    case CORETYPE_UNKNOWN:
+    default:
+      return "unknown";
+    }
+}
+
+const char *
+Token::get_type_hint_str () const
+{
+  return get_type_hint_string (type_hint);
+}
+
+const std::string &
+Token::get_str () const
+{
+  // FIXME: attempt to return null again
+  // gcc_assert(str != NULL);
+
+  // HACK: allow referencing an empty string
+  static const std::string empty = "";
+
+  if (str == NULL)
+    {
+      rust_error_at (get_locus (),
+		     "attempted to get string for %<%s%>, which has no string. "
+		     "returning empty string instead",
+		     get_token_description ());
+      return empty;
+    }
+  return *str;
+}
+} // namespace Rust
diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h
new file mode 100644
index 00000000000..3fa46a2cebe
--- /dev/null
+++ b/gcc/rust/lex/rust-token.h
@@ -0,0 +1,455 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_TOKEN_H
+#define RUST_TOKEN_H
+
+#include "rust-linemap.h"
+#include "rust-codepoint.h"
+
+// order: config, system, coretypes, input
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "input.h"
+
+namespace Rust {
+// "Primitive core types" in Rust - the different int and float types, as well
+// as some others
+enum PrimitiveCoreType
+{
+  CORETYPE_UNKNOWN,
+  // named primitives
+  CORETYPE_BOOL,
+  CORETYPE_CHAR,
+  CORETYPE_STR,
+  // okay technically int and uint are arch-dependent (pointer size)
+  CORETYPE_INT,
+  CORETYPE_UINT,
+  // numbered number primitives
+  CORETYPE_F32,
+  CORETYPE_F64,
+  CORETYPE_I8,
+  CORETYPE_I16,
+  CORETYPE_I32,
+  CORETYPE_I64,
+  CORETYPE_I128,
+  CORETYPE_U8,
+  CORETYPE_U16,
+  CORETYPE_U32,
+  CORETYPE_U64,
+  CORETYPE_U128,
+  // Pure decimals are used for tuple index.
+  // Also means there is no type hint.
+  CORETYPE_PURE_DECIMAL,
+  // arch-dependent pointer sizes
+  CORETYPE_ISIZE = CORETYPE_INT,
+  CORETYPE_USIZE = CORETYPE_UINT
+};
+
+// RS_TOKEN(name, description)
+// RS_TOKEN_KEYWORD(name, identifier)
+//
+// Keep RS_TOKEN_KEYWORD sorted
+
+/* note that abstract, async, become, box, do, final, macro, override, priv,
+ * try, typeof, unsized, virtual, and yield are unused */
+#define RS_TOKEN_LIST                                                          \
+  RS_TOKEN (FIRST_TOKEN, "<first-token-marker>")                               \
+  RS_TOKEN (END_OF_FILE, "end of file")                                        \
+  RS_TOKEN (EXCLAM, "!")                                                       \
+  RS_TOKEN (NOT_EQUAL, "!=")                                                   \
+  RS_TOKEN (PERCENT, "%")                                                      \
+  RS_TOKEN (PERCENT_EQ, "%=")                                                  \
+  RS_TOKEN (AMP, "&")                                                          \
+  RS_TOKEN (AMP_EQ, "&=")                                                      \
+  RS_TOKEN (LOGICAL_AND, "&&")                                                 \
+  RS_TOKEN (ASTERISK, "*")                                                     \
+  RS_TOKEN (ASTERISK_EQ, "*=")                                                 \
+  RS_TOKEN (PLUS, "+")                                                         \
+  RS_TOKEN (PLUS_EQ, "+=")                                                     \
+  RS_TOKEN (COMMA, ",")                                                        \
+  RS_TOKEN (MINUS, "-")                                                        \
+  RS_TOKEN (MINUS_EQ, "-=")                                                    \
+  RS_TOKEN (RETURN_TYPE, "->")                                                 \
+  RS_TOKEN (DOT, ".")                                                          \
+  RS_TOKEN (DOT_DOT, "..")                                                     \
+  RS_TOKEN (DOT_DOT_EQ, "..=")                                                 \
+  RS_TOKEN (ELLIPSIS, "...")                                                   \
+  RS_TOKEN (DIV, "/")                                                          \
+  RS_TOKEN (DIV_EQ, "/=")                                                      \
+  RS_TOKEN (COLON, ":")                                                        \
+  RS_TOKEN (SEMICOLON, ";")                                                    \
+  RS_TOKEN (LEFT_SHIFT, "<<")                                                  \
+  RS_TOKEN (LEFT_SHIFT_EQ, "<<=")                                              \
+  RS_TOKEN (LEFT_ANGLE, "<")                                                   \
+  RS_TOKEN (LESS_OR_EQUAL, "<=")                                               \
+  RS_TOKEN (EQUAL, "=")                                                        \
+  RS_TOKEN (EQUAL_EQUAL, "==")                                                 \
+  RS_TOKEN (MATCH_ARROW, "=>")                                                 \
+  RS_TOKEN (RIGHT_ANGLE, ">")                                                  \
+  RS_TOKEN (GREATER_OR_EQUAL, ">=")                                            \
+  RS_TOKEN (RIGHT_SHIFT, ">>")                                                 \
+  RS_TOKEN (RIGHT_SHIFT_EQ, ">>=")                                             \
+  RS_TOKEN (PATTERN_BIND, "@")                                                 \
+  RS_TOKEN (TILDE, "~")                                                        \
+  RS_TOKEN (BACKSLASH, "\\")                                                   \
+  RS_TOKEN (BACKTICK, "`")                                                     \
+  RS_TOKEN (CARET, "^")                                                        \
+  RS_TOKEN (CARET_EQ, "^=")                                                    \
+  RS_TOKEN (PIPE, "|")                                                         \
+  RS_TOKEN (PIPE_EQ, "|=")                                                     \
+  RS_TOKEN (OR, "||")                                                          \
+  RS_TOKEN (QUESTION_MARK, "?")                                                \
+  RS_TOKEN (HASH, "#")                                                         \
+  /* from here on, dodgy and may not be correct. not operators and may be      \
+   * symbols */                                                                \
+  /* RS_TOKEN(SPACE, " ") probably too dodgy */                                \
+  /* RS_TOKEN(NEWLINE, "\n")*/                                                 \
+  RS_TOKEN (SCOPE_RESOLUTION, "::") /* dodgy */                                \
+  RS_TOKEN (SINGLE_QUOTE, "'") /* should i differentiate from lifetime? */     \
+  RS_TOKEN (DOUBLE_QUOTE, "\"")                                                \
+  RS_TOKEN (UNDERSCORE,                                                        \
+	    "_") /* TODO: treat as reserved word like mrustc instead? */       \
+  RS_TOKEN (IDENTIFIER, "identifier")                                          \
+  RS_TOKEN (INT_LITERAL,                                                       \
+	    "integer literal") /* do different int and float types need        \
+				  different literal types? */                  \
+  RS_TOKEN (FLOAT_LITERAL, "float literal")                                    \
+  RS_TOKEN (STRING_LITERAL, "string literal")                                  \
+  RS_TOKEN (CHAR_LITERAL, "character literal")                                 \
+  RS_TOKEN (BYTE_STRING_LITERAL, "byte string literal")                        \
+  RS_TOKEN (BYTE_CHAR_LITERAL, "byte character literal")                       \
+  RS_TOKEN (LIFETIME, "lifetime") /* TODO: improve token type */               \
+  /* Have "interpolated" tokens (whatever that means)? identifer, path, type,  \
+   * pattern, */                                                               \
+  /* expression, statement, block, meta, item in mrustc (but not directly in   \
+   * lexer). */                                                                \
+  RS_TOKEN (LEFT_PAREN, "(")                                                   \
+  RS_TOKEN (RIGHT_PAREN, ")")                                                  \
+  RS_TOKEN (LEFT_CURLY, "{")                                                   \
+  RS_TOKEN (RIGHT_CURLY, "}")                                                  \
+  RS_TOKEN (LEFT_SQUARE, "[")                                                  \
+  RS_TOKEN (RIGHT_SQUARE, "]")                                                 \
+  /* Macros */                                                                 \
+  RS_TOKEN (DOLLAR_SIGN, "$")                                                  \
+  /* Doc Comments */                                                           \
+  RS_TOKEN (INNER_DOC_COMMENT, "#![doc]")                                      \
+  RS_TOKEN (OUTER_DOC_COMMENT, "#[doc]")                                       \
+  /* have "weak" union and 'static keywords? */                                \
+                                                                               \
+  RS_TOKEN_KEYWORD (ABSTRACT, "abstract") /* unused */                         \
+  RS_TOKEN_KEYWORD (AS, "as")                                                  \
+  RS_TOKEN_KEYWORD (ASYNC, "async")   /* unused */                             \
+  RS_TOKEN_KEYWORD (BECOME, "become") /* unused */                             \
+  RS_TOKEN_KEYWORD (BOX, "box")	      /* unused */                             \
+  RS_TOKEN_KEYWORD (BREAK, "break")                                            \
+  RS_TOKEN_KEYWORD (CONST, "const")                                            \
+  RS_TOKEN_KEYWORD (CONTINUE, "continue")                                      \
+  RS_TOKEN_KEYWORD (CRATE, "crate")                                            \
+  /* FIXME: Do we need to add $crate (DOLLAR_CRATE) as a reserved kw? */       \
+  RS_TOKEN_KEYWORD (DO, "do") /* unused */                                     \
+  RS_TOKEN_KEYWORD (DYN, "dyn")                                                \
+  RS_TOKEN_KEYWORD (ELSE, "else")                                              \
+  RS_TOKEN_KEYWORD (ENUM_TOK, "enum")                                          \
+  RS_TOKEN_KEYWORD (EXTERN_TOK, "extern")                                      \
+  RS_TOKEN_KEYWORD (FALSE_LITERAL, "false")                                    \
+  RS_TOKEN_KEYWORD (FINAL_TOK, "final") /* unused */                           \
+  RS_TOKEN_KEYWORD (FN_TOK, "fn")                                              \
+  RS_TOKEN_KEYWORD (FOR, "for")                                                \
+  RS_TOKEN_KEYWORD (IF, "if")                                                  \
+  RS_TOKEN_KEYWORD (IMPL, "impl")                                              \
+  RS_TOKEN_KEYWORD (IN, "in")                                                  \
+  RS_TOKEN_KEYWORD (LET, "let")                                                \
+  RS_TOKEN_KEYWORD (LOOP, "loop")                                              \
+  RS_TOKEN_KEYWORD (MACRO, "macro") /* unused */                               \
+  RS_TOKEN_KEYWORD (MATCH_TOK, "match")                                        \
+  RS_TOKEN_KEYWORD (MOD, "mod")                                                \
+  RS_TOKEN_KEYWORD (MOVE, "move")                                              \
+  RS_TOKEN_KEYWORD (MUT, "mut")                                                \
+  RS_TOKEN_KEYWORD (OVERRIDE_TOK, "override") /* unused */                     \
+  RS_TOKEN_KEYWORD (PRIV, "priv")	      /* unused */                     \
+  RS_TOKEN_KEYWORD (PUB, "pub")                                                \
+  RS_TOKEN_KEYWORD (REF, "ref")                                                \
+  RS_TOKEN_KEYWORD (RETURN_TOK, "return")                                      \
+  RS_TOKEN_KEYWORD (SELF_ALIAS,                                                \
+		    "Self") /* mrustc does not treat this as a reserved word*/ \
+  RS_TOKEN_KEYWORD (SELF, "self")                                              \
+  RS_TOKEN_KEYWORD (STATIC_TOK, "static")                                      \
+  RS_TOKEN_KEYWORD (STRUCT_TOK, "struct")                                      \
+  RS_TOKEN_KEYWORD (SUPER, "super")                                            \
+  RS_TOKEN_KEYWORD (TRAIT, "trait")                                            \
+  RS_TOKEN_KEYWORD (TRUE_LITERAL, "true")                                      \
+  RS_TOKEN_KEYWORD (TRY, "try") /* unused */                                   \
+  RS_TOKEN_KEYWORD (TYPE, "type")                                              \
+  RS_TOKEN_KEYWORD (TYPEOF, "typeof") /* unused */                             \
+  RS_TOKEN_KEYWORD (UNSAFE, "unsafe")                                          \
+  RS_TOKEN_KEYWORD (UNSIZED, "unsized") /* unused */                           \
+  RS_TOKEN_KEYWORD (USE, "use")                                                \
+  RS_TOKEN_KEYWORD (VIRTUAL, "virtual") /* unused */                           \
+  RS_TOKEN_KEYWORD (WHERE, "where")                                            \
+  RS_TOKEN_KEYWORD (WHILE, "while")                                            \
+  RS_TOKEN_KEYWORD (YIELD, "yield") /* unused */                               \
+                                                                               \
+  RS_TOKEN (LAST_TOKEN, "<last-token-marker>")
+
+// Contains all token types. Crappy implementation via x-macros.
+enum TokenId
+{
+#define RS_TOKEN(name, _) name,
+#define RS_TOKEN_KEYWORD(x, y) RS_TOKEN (x, y)
+  RS_TOKEN_LIST
+#undef RS_TOKEN_KEYWORD
+#undef RS_TOKEN
+};
+
+// dodgy "TokenPtr" declaration with Token forward declaration
+class Token;
+// A smart pointer (shared_ptr) to Token.
+typedef std::shared_ptr<Token> TokenPtr;
+// A smart pointer (shared_ptr) to a constant Token.
+typedef std::shared_ptr<const Token> const_TokenPtr;
+
+// Hackily defined way to get token description for enum value using x-macros
+const char *
+get_token_description (TokenId id);
+/* Hackily defined way to get token description as a string for enum value using
+ * x-macros */
+const char *
+token_id_to_str (TokenId id);
+// Get type hint description as a string.
+const char *
+get_type_hint_string (PrimitiveCoreType type);
+
+// Represents a single token. Create using factory static methods.
+class Token
+{
+private:
+  // Token kind.
+  TokenId token_id;
+  // Token location.
+  Location locus;
+  // Associated text (if any) of token.
+  std::unique_ptr<std::string> str;
+  // TODO: maybe remove issues and just store std::string as value?
+  /* Type hint for token based on lexer data (e.g. type suffix). Does not exist
+   * for most tokens. */
+  PrimitiveCoreType type_hint;
+
+  // Token constructor from token id and location. Has a null string.
+  Token (TokenId token_id, Location location)
+    : token_id (token_id), locus (location), str (nullptr),
+      type_hint (CORETYPE_UNKNOWN)
+  {}
+
+  // Token constructor from token id, location, and a string.
+  Token (TokenId token_id, Location location, std::string &&paramStr)
+    : token_id (token_id), locus (location),
+      str (new std::string (std::move (paramStr))), type_hint (CORETYPE_UNKNOWN)
+  {}
+
+  // Token constructor from token id, location, and a char.
+  Token (TokenId token_id, Location location, char paramChar)
+    : token_id (token_id), locus (location),
+      str (new std::string (1, paramChar)), type_hint (CORETYPE_UNKNOWN)
+  {}
+
+  // Token constructor from token id, location, and a "codepoint".
+  Token (TokenId token_id, Location location, Codepoint paramCodepoint)
+    : token_id (token_id), locus (location),
+      str (new std::string (paramCodepoint.as_string ())),
+      type_hint (CORETYPE_UNKNOWN)
+  {}
+
+  // Token constructor from token id, location, a string, and type hint.
+  Token (TokenId token_id, Location location, std::string &&paramStr,
+	 PrimitiveCoreType parType)
+    : token_id (token_id), locus (location),
+      str (new std::string (std::move (paramStr))), type_hint (parType)
+  {}
+
+public:
+  // No default constructor.
+  Token () = delete;
+  // Do not copy/assign tokens.
+  Token (const Token &) = delete;
+  Token &operator= (const Token &) = delete;
+
+  // Allow moving tokens.
+  Token (Token &&other) = default;
+  Token &operator= (Token &&other) = default;
+
+  ~Token () = default;
+
+  /* TODO: make_shared (which saves a heap allocation) does not work with the
+   * private constructor */
+
+  // Makes and returns a new TokenPtr (with null string).
+  static TokenPtr make (TokenId token_id, Location locus)
+  {
+    // return std::make_shared<Token> (token_id, locus);
+    return TokenPtr (new Token (token_id, locus));
+  }
+
+  // Makes and returns a new TokenPtr of type IDENTIFIER.
+  static TokenPtr make_identifier (Location locus, std::string &&str)
+  {
+    // return std::make_shared<Token> (IDENTIFIER, locus, str);
+    return TokenPtr (new Token (IDENTIFIER, locus, std::move (str)));
+  }
+
+  // Makes and returns a new TokenPtr of type INT_LITERAL.
+  static TokenPtr make_int (Location locus, std::string &&str,
+			    PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
+  {
+    // return std::make_shared<Token> (INT_LITERAL, locus, str, type_hint);
+    return TokenPtr (
+      new Token (INT_LITERAL, locus, std::move (str), type_hint));
+  }
+
+  // Makes and returns a new TokenPtr of type FLOAT_LITERAL.
+  static TokenPtr make_float (Location locus, std::string &&str,
+			      PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
+  {
+    // return std::make_shared<Token> (FLOAT_LITERAL, locus, str, type_hint);
+    return TokenPtr (
+      new Token (FLOAT_LITERAL, locus, std::move (str), type_hint));
+  }
+
+  // Makes and returns a new TokenPtr of type STRING_LITERAL.
+  static TokenPtr make_string (Location locus, std::string &&str)
+  {
+    // return std::make_shared<Token> (STRING_LITERAL, locus, str,
+    // CORETYPE_STR);
+    return TokenPtr (
+      new Token (STRING_LITERAL, locus, std::move (str), CORETYPE_STR));
+  }
+
+  // Makes and returns a new TokenPtr of type CHAR_LITERAL.
+  static TokenPtr make_char (Location locus, Codepoint char_lit)
+  {
+    // return std::make_shared<Token> (CHAR_LITERAL, locus, char_lit);
+    return TokenPtr (new Token (CHAR_LITERAL, locus, char_lit));
+  }
+
+  // Makes and returns a new TokenPtr of type BYTE_CHAR_LITERAL.
+  static TokenPtr make_byte_char (Location locus, char byte_char)
+  {
+    // return std::make_shared<Token> (BYTE_CHAR_LITERAL, locus, byte_char);
+    return TokenPtr (new Token (BYTE_CHAR_LITERAL, locus, byte_char));
+  }
+
+  // Makes and returns a new TokenPtr of type BYTE_STRING_LITERAL (fix).
+  static TokenPtr make_byte_string (Location locus, std::string &&str)
+  {
+    // return std::make_shared<Token> (BYTE_STRING_LITERAL, locus, str);
+    return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, std::move (str)));
+  }
+
+  // Makes and returns a new TokenPtr of type INNER_DOC_COMMENT.
+  static TokenPtr make_inner_doc_comment (Location locus, std::string &&str)
+  {
+    return TokenPtr (new Token (INNER_DOC_COMMENT, locus, std::move (str)));
+  }
+
+  // Makes and returns a new TokenPtr of type OUTER_DOC_COMMENT.
+  static TokenPtr make_outer_doc_comment (Location locus, std::string &&str)
+  {
+    return TokenPtr (new Token (OUTER_DOC_COMMENT, locus, std::move (str)));
+  }
+
+  // Makes and returns a new TokenPtr of type LIFETIME.
+  static TokenPtr make_lifetime (Location locus, std::string &&str)
+  {
+    // return std::make_shared<Token> (LIFETIME, locus, str);
+    return TokenPtr (new Token (LIFETIME, locus, std::move (str)));
+  }
+
+  // Gets id of the token.
+  TokenId get_id () const { return token_id; }
+
+  // Gets location of the token.
+  Location get_locus () const { return locus; }
+
+  // Gets string description of the token.
+  const std::string &
+  get_str () const; /*{
+// FIXME: put in header again when fix null problem
+//gcc_assert(str != nullptr);
+if (str == nullptr) {
+error_at(get_locus(), "attempted to get string for '%s', which has no string.
+returning empty string instead.", get_token_description()); return "";
+}
+return *str;
+}*/
+
+  // Gets token's type hint info.
+  PrimitiveCoreType get_type_hint () const
+  {
+    return type_hint == CORETYPE_PURE_DECIMAL ? CORETYPE_UNKNOWN : type_hint;
+  }
+
+  // diagnostics (error reporting)
+  const char *get_token_description () const
+  {
+    return Rust::get_token_description (token_id);
+  }
+
+  // debugging
+  const char *token_id_to_str () const
+  {
+    return Rust::token_id_to_str (token_id);
+  }
+
+  // debugging
+  const char *get_type_hint_str () const;
+
+  /* Returns whether the token is a literal of any type (int, float, char,
+   * string, byte char, byte string). */
+  bool is_literal () const
+  {
+    switch (token_id)
+      {
+      case INT_LITERAL:
+      case FLOAT_LITERAL:
+      case CHAR_LITERAL:
+      case STRING_LITERAL:
+      case BYTE_CHAR_LITERAL:
+      case BYTE_STRING_LITERAL:
+	return true;
+      default:
+	return false;
+      }
+  }
+
+  /* Returns whether the token actually has a string (regardless of whether it
+   * should or not). */
+  bool has_str () const { return str != nullptr; }
+
+  // Returns whether the token should have a string.
+  bool should_have_str () const
+  {
+    return is_literal () || token_id == IDENTIFIER || token_id == LIFETIME;
+  }
+
+  // Returns whether the token is a pure decimal int literal
+  bool is_pure_decimal () const { return type_hint == CORETYPE_PURE_DECIMAL; }
+};
+} // namespace Rust
+
+#endif
diff --git a/gcc/rust/rust-buffered-queue.h b/gcc/rust/rust-buffered-queue.h
new file mode 100644
index 00000000000..afcc4670cac
--- /dev/null
+++ b/gcc/rust/rust-buffered-queue.h
@@ -0,0 +1,204 @@
+// Copyright (C) 2020-2022 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_BUFFERED_QUEUE_H
+#define RUST_BUFFERED_QUEUE_H
+
+#include "rust-system.h"
+
+namespace Rust {
+/* Buffered queue implementation. Items are of type T, queue source is of type
+ * Source. Note that this is owning of the source. */
+template <typename T, typename Source> class buffered_queue
+{
+public:
+  // Construct empty queue from Source src.
+  buffered_queue (Source src) : source (src), start (0), end (0), buffer () {}
+
+  /* disable copying (since source is probably non-copyable)
+   * TODO is this actually a good idea? If source is non-copyable, it would
+   * just delete the copy constructor anyway.*/
+  buffered_queue (const buffered_queue &other) = delete;
+  buffered_queue &operator= (const buffered_queue &other) = delete;
+
+  // enable moving
+  buffered_queue (buffered_queue &&other) = default;
+  buffered_queue &operator= (buffered_queue &&other) = default;
+
+  // Returns token at position start + n (i.e. n tokens ahead).
+  T peek (int n)
+  {
+    // n should not be behind
+    rust_assert (n >= 0);
+
+    int num_queued_items = end - start;
+    int num_items_required = n + 1;
+
+    // if required items go past end of queue, add them to queue
+    if (num_items_required > num_queued_items)
+      {
+	int num_items_to_read = num_items_required - num_queued_items;
+
+	/* if queue length + extra items is larger than buffer size, expand
+	 * buffer */
+	if (end + num_items_to_read > (int) buffer.size ())
+	  {
+	    // Resize the buffer by 1.5x
+	    int new_size = (buffer.size () + num_items_to_read);
+	    new_size += (new_size >> 1);
+
+	    // old method:
+	    /*
+		  // create new queue buffer with new size
+		  std::vector<T> new_queue (new_size);
+		  std::copy (buffer.begin () + start, buffer.begin () + end,
+			     new_queue.begin ());
+		  start = 0;
+		  end = num_queued_items;
+		  // TODO: would move be better here? optimisation for move with
+		  // shared pointer?
+
+		  // swap member buffer and new queue buffer
+		  std::swap (buffer, new_queue);
+	    */
+
+	    // TODO: determine overhead of this approach vs copy. Should be
+	    // lower.
+	    std::vector<T> new_queue;
+	    new_queue.reserve (new_size);
+	    new_queue.insert (new_queue.begin (),
+			      std::make_move_iterator (buffer.begin () + start),
+			      std::make_move_iterator (buffer.begin () + end));
+	    start = 0;
+	    end = num_queued_items;
+	    // fill up rest of vector with junk so that indexing can work
+	    new_queue.insert (new_queue.begin () + end,
+			      new_size - new_queue.size (), T ());
+
+	    buffer = std::move (new_queue);
+	    /* this should be best method - std::move(range) would have
+	     * allocation problems; initial construction would require
+	     * reallocation upon resizing */
+
+	    // validate that buffer is large enough now
+	    rust_assert (end + num_items_to_read <= (int) buffer.size ());
+	  }
+
+	/* iterate through buffer and invoke operator () on source on values
+	 * past original end */
+	for (int i = 0; i < num_items_to_read; i++)
+	  buffer[end + i] = source.next ();
+
+	// move end based on additional items added
+	end += num_items_to_read;
+      }
+
+    rust_assert (0 <= start);
+    rust_assert (start <= end);
+    rust_assert (end <= (int) buffer.size ());
+
+    rust_assert (start + n < end);
+
+    // return value at start + n in buffer
+    return buffer[start + n];
+  }
+
+  /* TODO: add faster peek current token to remove overhead of conditional
+   * branches? */
+
+  // Advances start by n + 1.
+  void skip (int n)
+  {
+    // Call peek to ensure requested n is actually in queue.
+    peek (n);
+
+    // Clear queue values from start to n (inclusive).
+    for (int i = 0; i < (n + 1); i++)
+      buffer[start + i] = T ();
+
+    // Move start forward by n + 1.
+    start += (n + 1);
+
+    // Ensure start is not impossible somehow
+    rust_assert (0 <= start);
+    rust_assert (start <= end);
+
+    // Compact buffer if empty
+    if (start == end)
+      start = end = 0;
+  }
+
+  /* Inserts element at front of vector. Really dirty hack with terrible
+   * performance, only use when really needed. */
+  void insert_at_front (T elem_to_insert)
+  {
+    // TODO: test as this may not work properly
+
+    // Insert actual element in buffer at start.
+    buffer.insert (buffer.begin (), elem_to_insert);
+
+    /* Increase the end number since added element means all others have shifted
+     * one along */
+    end++;
+  }
+
+  // Insert at arbitrary position (attempt)
+  void insert (int index, T elem_to_insert)
+  {
+    // TODO: test as this may not work properly
+
+    // n should not be behind
+    rust_assert (index >= 0);
+
+    // call peek to ensure that the items behind this (at least) are in queue
+    if (index >= 1)
+      peek (index - 1);
+    else
+      peek (index);
+
+    buffer.insert (buffer.begin () + start + index, std::move (elem_to_insert));
+
+    end++;
+  }
+
+  // Replaces the current value in the buffer. Total HACK.
+  void replace_current_value (T replacement)
+  {
+    // call peek to ensure value exists
+    peek (0);
+
+    buffer[start] = std::move (replacement);
+
+    // don't move start or end
+  }
+
+private:
+  // Source of tokens for queue.
+  Source source;
+
+  // Begin of range in buffer, inclusive.
+  int start;
+  // End of range in buffer, exclusive.
+  int end;
+
+  // Queue buffer.
+  std::vector<T> buffer;
+};
+} // namespace Rust
+
+#endif
-- 
2.25.1



More information about the Gcc-rust mailing list