Introduce rustc_lexer

The idea here is to make a reusable library out of the existing
rust-lexer, by separating out pure lexing and rustc-specific concerns,
like spans, error reporting an interning.

So, rustc_lexer operates directly on `&str`, produces simple tokens
which are a pair of type-tag and a bit of original text, and does not
report errors, instead storing them as flags on the token.

Cargo.lock

@@ -2972,6 +2972,13 @@ dependencies = [
  "tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rustc_lexer"
+version = "0.1.0"
+dependencies = [
+ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rustc_lint"
 version = "0.0.0"
@@ -3622,6 +3629,7 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_data_structures 0.0.0",
  "rustc_errors 0.0.0",
+ "rustc_lexer 0.1.0",
  "rustc_macros 0.1.0",
  "rustc_target 0.0.0",
  "scoped-tls 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",

src/librustc_lexer/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_lexer"
+version = "0.1.0"
+edition = "2018"
+
+# Note that this crate purposefully does not depend on other rustc crates
+[dependencies]
+unicode-xid = { version = "0.1.0", optional = true }

src/librustc_lexer/src/cursor.rs

@@ -0,0 +1,57 @@
+use std::str::Chars;
+
+pub(crate) struct Cursor<'a> {
+    initial_len: usize,
+    chars: Chars<'a>,
+    #[cfg(debug_assertions)]
+    prev: char,
+}
+
+pub(crate) const EOF_CHAR: char = '\0';
+
+impl<'a> Cursor<'a> {
+    pub(crate) fn new(input: &'a str) -> Cursor<'a> {
+        Cursor {
+            initial_len: input.len(),
+            chars: input.chars(),
+            #[cfg(debug_assertions)]
+            prev: EOF_CHAR,
+        }
+    }
+    /// For debug assertions only
+    pub(crate) fn prev(&self) -> char {
+        #[cfg(debug_assertions)]
+        {
+            self.prev
+        }
+
+        #[cfg(not(debug_assertions))]
+        {
+            '\0'
+        }
+    }
+    pub(crate) fn nth_char(&self, n: usize) -> char {
+        self.chars().nth(n).unwrap_or(EOF_CHAR)
+    }
+    pub(crate) fn is_eof(&self) -> bool {
+        self.chars.as_str().is_empty()
+    }
+    pub(crate) fn len_consumed(&self) -> usize {
+        self.initial_len - self.chars.as_str().len()
+    }
+    /// Returns an iterator over the remaining characters.
+    fn chars(&self) -> Chars<'a> {
+        self.chars.clone()
+    }
+    /// Moves to the next character.
+    pub(crate) fn bump(&mut self) -> Option<char> {
+        let c = self.chars.next()?;
+
+        #[cfg(debug_assertions)]
+        {
+            self.prev = c;
+        }
+
+        Some(c)
+    }
+}