aboutsummaryrefslogtreecommitdiff
path: root/proc-macro2/src/fallback.rs
diff options
context:
space:
mode:
Diffstat (limited to 'proc-macro2/src/fallback.rs')
-rw-r--r--proc-macro2/src/fallback.rs1458
1 files changed, 1458 insertions, 0 deletions
diff --git a/proc-macro2/src/fallback.rs b/proc-macro2/src/fallback.rs
new file mode 100644
index 0000000..9762d3b
--- /dev/null
+++ b/proc-macro2/src/fallback.rs
@@ -0,0 +1,1458 @@
+#[cfg(span_locations)]
+use std::cell::RefCell;
+#[cfg(span_locations)]
+use std::cmp;
+use std::fmt;
+use std::iter;
+use std::ops::RangeBounds;
+#[cfg(procmacro2_semver_exempt)]
+use std::path::Path;
+use std::path::PathBuf;
+use std::str::FromStr;
+use std::vec;
+
+use crate::strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
+use crate::{Delimiter, Punct, Spacing, TokenTree};
+use unicode_xid::UnicodeXID;
+
+#[derive(Clone)]
+pub struct TokenStream {
+ inner: Vec<TokenTree>,
+}
+
+#[derive(Debug)]
+pub struct LexError;
+
+impl TokenStream {
+ pub fn new() -> TokenStream {
+ TokenStream { inner: Vec::new() }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.inner.len() == 0
+ }
+}
+
+#[cfg(span_locations)]
+fn get_cursor(src: &str) -> Cursor {
+ // Create a dummy file & add it to the source map
+ SOURCE_MAP.with(|cm| {
+ let mut cm = cm.borrow_mut();
+ let name = format!("<parsed string {}>", cm.files.len());
+ let span = cm.add_file(&name, src);
+ Cursor {
+ rest: src,
+ off: span.lo,
+ }
+ })
+}
+
+#[cfg(not(span_locations))]
+fn get_cursor(src: &str) -> Cursor {
+ Cursor { rest: src }
+}
+
+impl FromStr for TokenStream {
+ type Err = LexError;
+
+ fn from_str(src: &str) -> Result<TokenStream, LexError> {
+ // Create a dummy file & add it to the source map
+ let cursor = get_cursor(src);
+
+ match token_stream(cursor) {
+ Ok((input, output)) => {
+ if skip_whitespace(input).len() != 0 {
+ Err(LexError)
+ } else {
+ Ok(output)
+ }
+ }
+ Err(LexError) => Err(LexError),
+ }
+ }
+}
+
+impl fmt::Display for TokenStream {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ let mut joint = false;
+ for (i, tt) in self.inner.iter().enumerate() {
+ if i != 0 && !joint {
+ write!(f, " ")?;
+ }
+ joint = false;
+ match *tt {
+ TokenTree::Group(ref tt) => {
+ let (start, end) = match tt.delimiter() {
+ Delimiter::Parenthesis => ("(", ")"),
+ Delimiter::Brace => ("{", "}"),
+ Delimiter::Bracket => ("[", "]"),
+ Delimiter::None => ("", ""),
+ };
+ if tt.stream().into_iter().next().is_none() {
+ write!(f, "{} {}", start, end)?
+ } else {
+ write!(f, "{} {} {}", start, tt.stream(), end)?
+ }
+ }
+ TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
+ TokenTree::Punct(ref tt) => {
+ write!(f, "{}", tt.as_char())?;
+ match tt.spacing() {
+ Spacing::Alone => {}
+ Spacing::Joint => joint = true,
+ }
+ }
+ TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
+ }
+ }
+
+ Ok(())
+ }
+}
+
+impl fmt::Debug for TokenStream {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ f.write_str("TokenStream ")?;
+ f.debug_list().entries(self.clone()).finish()
+ }
+}
+
+#[cfg(use_proc_macro)]
+impl From<proc_macro::TokenStream> for TokenStream {
+ fn from(inner: proc_macro::TokenStream) -> TokenStream {
+ inner
+ .to_string()
+ .parse()
+ .expect("compiler token stream parse failed")
+ }
+}
+
+#[cfg(use_proc_macro)]
+impl From<TokenStream> for proc_macro::TokenStream {
+ fn from(inner: TokenStream) -> proc_macro::TokenStream {
+ inner
+ .to_string()
+ .parse()
+ .expect("failed to parse to compiler tokens")
+ }
+}
+
+impl From<TokenTree> for TokenStream {
+ fn from(tree: TokenTree) -> TokenStream {
+ TokenStream { inner: vec![tree] }
+ }
+}
+
+impl iter::FromIterator<TokenTree> for TokenStream {
+ fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
+ let mut v = Vec::new();
+
+ for token in streams.into_iter() {
+ v.push(token);
+ }
+
+ TokenStream { inner: v }
+ }
+}
+
+impl iter::FromIterator<TokenStream> for TokenStream {
+ fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
+ let mut v = Vec::new();
+
+ for stream in streams.into_iter() {
+ v.extend(stream.inner);
+ }
+
+ TokenStream { inner: v }
+ }
+}
+
+impl Extend<TokenTree> for TokenStream {
+ fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
+ self.inner.extend(streams);
+ }
+}
+
+impl Extend<TokenStream> for TokenStream {
+ fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
+ self.inner
+ .extend(streams.into_iter().flat_map(|stream| stream));
+ }
+}
+
+pub type TokenTreeIter = vec::IntoIter<TokenTree>;
+
+impl IntoIterator for TokenStream {
+ type Item = TokenTree;
+ type IntoIter = TokenTreeIter;
+
+ fn into_iter(self) -> TokenTreeIter {
+ self.inner.into_iter()
+ }
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub struct SourceFile {
+ path: PathBuf,
+}
+
+impl SourceFile {
+ /// Get the path to this source file as a string.
+ pub fn path(&self) -> PathBuf {
+ self.path.clone()
+ }
+
+ pub fn is_real(&self) -> bool {
+ // XXX(nika): Support real files in the future?
+ false
+ }
+}
+
+impl fmt::Debug for SourceFile {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ f.debug_struct("SourceFile")
+ .field("path", &self.path())
+ .field("is_real", &self.is_real())
+ .finish()
+ }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct LineColumn {
+ pub line: usize,
+ pub column: usize,
+}
+
+#[cfg(span_locations)]
+thread_local! {
+ static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
+ // NOTE: We start with a single dummy file which all call_site() and
+ // def_site() spans reference.
+ files: vec![{
+ #[cfg(procmacro2_semver_exempt)]
+ {
+ FileInfo {
+ name: "<unspecified>".to_owned(),
+ span: Span { lo: 0, hi: 0 },
+ lines: vec![0],
+ }
+ }
+
+ #[cfg(not(procmacro2_semver_exempt))]
+ {
+ FileInfo {
+ span: Span { lo: 0, hi: 0 },
+ lines: vec![0],
+ }
+ }
+ }],
+ });
+}
+
+#[cfg(span_locations)]
+struct FileInfo {
+ #[cfg(procmacro2_semver_exempt)]
+ name: String,
+ span: Span,
+ lines: Vec<usize>,
+}
+
+#[cfg(span_locations)]
+impl FileInfo {
+ fn offset_line_column(&self, offset: usize) -> LineColumn {
+ assert!(self.span_within(Span {
+ lo: offset as u32,
+ hi: offset as u32
+ }));
+ let offset = offset - self.span.lo as usize;
+ match self.lines.binary_search(&offset) {
+ Ok(found) => LineColumn {
+ line: found + 1,
+ column: 0,
+ },
+ Err(idx) => LineColumn {
+ line: idx,
+ column: offset - self.lines[idx - 1],
+ },
+ }
+ }
+
+ fn span_within(&self, span: Span) -> bool {
+ span.lo >= self.span.lo && span.hi <= self.span.hi
+ }
+}
+
+/// Computesthe offsets of each line in the given source string.
+#[cfg(span_locations)]
+fn lines_offsets(s: &str) -> Vec<usize> {
+ let mut lines = vec![0];
+ let mut prev = 0;
+ while let Some(len) = s[prev..].find('\n') {
+ prev += len + 1;
+ lines.push(prev);
+ }
+ lines
+}
+
+#[cfg(span_locations)]
+struct SourceMap {
+ files: Vec<FileInfo>,
+}
+
+#[cfg(span_locations)]
+impl SourceMap {
+ fn next_start_pos(&self) -> u32 {
+ // Add 1 so there's always space between files.
+ //
+ // We'll always have at least 1 file, as we initialize our files list
+ // with a dummy file.
+ self.files.last().unwrap().span.hi + 1
+ }
+
+ fn add_file(&mut self, name: &str, src: &str) -> Span {
+ let lines = lines_offsets(src);
+ let lo = self.next_start_pos();
+ // XXX(nika): Shouild we bother doing a checked cast or checked add here?
+ let span = Span {
+ lo,
+ hi: lo + (src.len() as u32),
+ };
+
+ #[cfg(procmacro2_semver_exempt)]
+ self.files.push(FileInfo {
+ name: name.to_owned(),
+ span,
+ lines,
+ });
+
+ #[cfg(not(procmacro2_semver_exempt))]
+ self.files.push(FileInfo { span, lines });
+ let _ = name;
+
+ span
+ }
+
+ fn fileinfo(&self, span: Span) -> &FileInfo {
+ for file in &self.files {
+ if file.span_within(span) {
+ return file;
+ }
+ }
+ panic!("Invalid span with no related FileInfo!");
+ }
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub struct Span {
+ #[cfg(span_locations)]
+ lo: u32,
+ #[cfg(span_locations)]
+ hi: u32,
+}
+
+impl Span {
+ #[cfg(not(span_locations))]
+ pub fn call_site() -> Span {
+ Span {}
+ }
+
+ #[cfg(span_locations)]
+ pub fn call_site() -> Span {
+ Span { lo: 0, hi: 0 }
+ }
+
+ #[cfg(procmacro2_semver_exempt)]
+ pub fn def_site() -> Span {
+ Span::call_site()
+ }
+
+ #[cfg(procmacro2_semver_exempt)]
+ pub fn resolved_at(&self, _other: Span) -> Span {
+ // Stable spans consist only of line/column information, so
+ // `resolved_at` and `located_at` only select which span the
+ // caller wants line/column information from.
+ *self
+ }
+
+ #[cfg(procmacro2_semver_exempt)]
+ pub fn located_at(&self, other: Span) -> Span {
+ other
+ }
+
+ #[cfg(procmacro2_semver_exempt)]
+ pub fn source_file(&self) -> SourceFile {
+ SOURCE_MAP.with(|cm| {
+ let cm = cm.borrow();
+ let fi = cm.fileinfo(*self);
+ SourceFile {
+ path: Path::new(&fi.name).to_owned(),
+ }
+ })
+ }
+
+ #[cfg(span_locations)]
+ pub fn start(&self) -> LineColumn {
+ SOURCE_MAP.with(|cm| {
+ let cm = cm.borrow();
+ let fi = cm.fileinfo(*self);
+ fi.offset_line_column(self.lo as usize)
+ })
+ }
+
+ #[cfg(span_locations)]
+ pub fn end(&self) -> LineColumn {
+ SOURCE_MAP.with(|cm| {
+ let cm = cm.borrow();
+ let fi = cm.fileinfo(*self);
+ fi.offset_line_column(self.hi as usize)
+ })
+ }
+
+ #[cfg(not(span_locations))]
+ pub fn join(&self, _other: Span) -> Option<Span> {
+ Some(Span {})
+ }
+
+ #[cfg(span_locations)]
+ pub fn join(&self, other: Span) -> Option<Span> {
+ SOURCE_MAP.with(|cm| {
+ let cm = cm.borrow();
+ // If `other` is not within the same FileInfo as us, return None.
+ if !cm.fileinfo(*self).span_within(other) {
+ return None;
+ }
+ Some(Span {
+ lo: cmp::min(self.lo, other.lo),
+ hi: cmp::max(self.hi, other.hi),
+ })
+ })
+ }
+
+ #[cfg(not(span_locations))]
+ fn first_byte(self) -> Self {
+ self
+ }
+
+ #[cfg(span_locations)]
+ fn first_byte(self) -> Self {
+ Span {
+ lo: self.lo,
+ hi: cmp::min(self.lo.saturating_add(1), self.hi),
+ }
+ }
+
+ #[cfg(not(span_locations))]
+ fn last_byte(self) -> Self {
+ self
+ }
+
+ #[cfg(span_locations)]
+ fn last_byte(self) -> Self {
+ Span {
+ lo: cmp::max(self.hi.saturating_sub(1), self.lo),
+ hi: self.hi,
+ }
+ }
+}
+
+impl fmt::Debug for Span {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ #[cfg(procmacro2_semver_exempt)]
+ return write!(f, "bytes({}..{})", self.lo, self.hi);
+
+ #[cfg(not(procmacro2_semver_exempt))]
+ write!(f, "Span")
+ }
+}
+
+pub fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
+ if cfg!(procmacro2_semver_exempt) {
+ debug.field("span", &span);
+ }
+}
+
+#[derive(Clone)]
+pub struct Group {
+ delimiter: Delimiter,
+ stream: TokenStream,
+ span: Span,
+}
+
+impl Group {
+ pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
+ Group {
+ delimiter,
+ stream,
+ span: Span::call_site(),
+ }
+ }
+
+ pub fn delimiter(&self) -> Delimiter {
+ self.delimiter
+ }
+
+ pub fn stream(&self) -> TokenStream {
+ self.stream.clone()
+ }
+
+ pub fn span(&self) -> Span {
+ self.span
+ }
+
+ pub fn span_open(&self) -> Span {
+ self.span.first_byte()
+ }
+
+ pub fn span_close(&self) -> Span {
+ self.span.last_byte()
+ }
+
+ pub fn set_span(&mut self, span: Span) {
+ self.span = span;
+ }
+}
+
+impl fmt::Display for Group {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ let (left, right) = match self.delimiter {
+ Delimiter::Parenthesis => ("(", ")"),
+ Delimiter::Brace => ("{", "}"),
+ Delimiter::Bracket => ("[", "]"),
+ Delimiter::None => ("", ""),
+ };
+
+ f.write_str(left)?;
+ self.stream.fmt(f)?;
+ f.write_str(right)?;
+
+ Ok(())
+ }
+}
+
+impl fmt::Debug for Group {
+ fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+ let mut debug = fmt.debug_struct("Group");
+ debug.field("delimiter", &self.delimiter);
+ debug.field("stream", &self.stream);
+ #[cfg(procmacro2_semver_exempt)]
+ debug.field("span", &self.span);
+ debug.finish()
+ }
+}
+
+#[derive(Clone)]
+pub struct Ident {
+ sym: String,
+ span: Span,
+ raw: bool,
+}
+
+impl Ident {
+ fn _new(string: &str, raw: bool, span: Span) -> Ident {
+ validate_ident(string);
+
+ Ident {
+ sym: string.to_owned(),
+ span,
+ raw,
+ }
+ }
+
+ pub fn new(string: &str, span: Span) -> Ident {
+ Ident::_new(string, false, span)
+ }
+
+ pub fn new_raw(string: &str, span: Span) -> Ident {
+ Ident::_new(string, true, span)
+ }
+
+ pub fn span(&self) -> Span {
+ self.span
+ }
+
+ pub fn set_span(&mut self, span: Span) {
+ self.span = span;
+ }
+}
+
+fn is_ident_start(c: char) -> bool {
+ ('a' <= c && c <= 'z')
+ || ('A' <= c && c <= 'Z')
+ || c == '_'
+ || (c > '\x7f' && UnicodeXID::is_xid_start(c))
+}
+
+fn is_ident_continue(c: char) -> bool {
+ ('a' <= c && c <= 'z')
+ || ('A' <= c && c <= 'Z')
+ || c == '_'
+ || ('0' <= c && c <= '9')
+ || (c > '\x7f' && UnicodeXID::is_xid_continue(c))
+}
+
+fn validate_ident(string: &str) {
+ let validate = string;
+ if validate.is_empty() {
+ panic!("Ident is not allowed to be empty; use Option<Ident>");
+ }
+
+ if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
+ panic!("Ident cannot be a number; use Literal instead");
+ }
+
+ fn ident_ok(string: &str) -> bool {
+ let mut chars = string.chars();
+ let first = chars.next().unwrap();
+ if !is_ident_start(first) {
+ return false;
+ }
+ for ch in chars {
+ if !is_ident_continue(ch) {
+ return false;
+ }
+ }
+ true
+ }
+
+ if !ident_ok(validate) {
+ panic!("{:?} is not a valid Ident", string);
+ }
+}
+
+impl PartialEq for Ident {
+ fn eq(&self, other: &Ident) -> bool {
+ self.sym == other.sym && self.raw == other.raw
+ }
+}
+
+impl<T> PartialEq<T> for Ident
+where
+ T: ?Sized + AsRef<str>,
+{
+ fn eq(&self, other: &T) -> bool {
+ let other = other.as_ref();
+ if self.raw {
+ other.starts_with("r#") && self.sym == other[2..]
+ } else {
+ self.sym == other
+ }
+ }
+}
+
+impl fmt::Display for Ident {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ if self.raw {
+ "r#".fmt(f)?;
+ }
+ self.sym.fmt(f)
+ }
+}
+
+impl fmt::Debug for Ident {
+ // Ident(proc_macro), Ident(r#union)
+ #[cfg(not(procmacro2_semver_exempt))]
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ let mut debug = f.debug_tuple("Ident");
+ debug.field(&format_args!("{}", self));
+ debug.finish()
+ }
+
+ // Ident {
+ // sym: proc_macro,
+ // span: bytes(128..138)
+ // }
+ #[cfg(procmacro2_semver_exempt)]
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ let mut debug = f.debug_struct("Ident");
+ debug.field("sym", &format_args!("{}", self));
+ debug.field("span", &self.span);
+ debug.finish()
+ }
+}
+
+#[derive(Clone)]
+pub struct Literal {
+ text: String,
+ span: Span,
+}
+
+macro_rules! suffixed_numbers {
+ ($($name:ident => $kind:ident,)*) => ($(
+ pub fn $name(n: $kind) -> Literal {
+ Literal::_new(format!(concat!("{}", stringify!($kind)), n))
+ }
+ )*)
+}
+
+macro_rules! unsuffixed_numbers {
+ ($($name:ident => $kind:ident,)*) => ($(
+ pub fn $name(n: $kind) -> Literal {
+ Literal::_new(n.to_string())
+ }
+ )*)
+}
+
+impl Literal {
+ fn _new(text: String) -> Literal {
+ Literal {
+ text,
+ span: Span::call_site(),
+ }
+ }
+
+ suffixed_numbers! {
+ u8_suffixed => u8,
+ u16_suffixed => u16,
+ u32_suffixed => u32,
+ u64_suffixed => u64,
+ u128_suffixed => u128,
+ usize_suffixed => usize,
+ i8_suffixed => i8,
+ i16_suffixed => i16,
+ i32_suffixed => i32,
+ i64_suffixed => i64,
+ i128_suffixed => i128,
+ isize_suffixed => isize,
+
+ f32_suffixed => f32,
+ f64_suffixed => f64,
+ }
+
+ unsuffixed_numbers! {
+ u8_unsuffixed => u8,
+ u16_unsuffixed => u16,
+ u32_unsuffixed => u32,
+ u64_unsuffixed => u64,
+ u128_unsuffixed => u128,
+ usize_unsuffixed => usize,
+ i8_unsuffixed => i8,
+ i16_unsuffixed => i16,
+ i32_unsuffixed => i32,
+ i64_unsuffixed => i64,
+ i128_unsuffixed => i128,
+ isize_unsuffixed => isize,
+ }
+
+ pub fn f32_unsuffixed(f: f32) -> Literal {
+ let mut s = f.to_string();
+ if !s.contains(".") {
+ s.push_str(".0");
+ }
+ Literal::_new(s)
+ }
+
+ pub fn f64_unsuffixed(f: f64) -> Literal {
+ let mut s = f.to_string();
+ if !s.contains(".") {
+ s.push_str(".0");
+ }
+ Literal::_new(s)
+ }
+
+ pub fn string(t: &str) -> Literal {
+ let mut text = String::with_capacity(t.len() + 2);
+ text.push('"');
+ for c in t.chars() {
+ if c == '\'' {
+ // escape_default turns this into "\'" which is unnecessary.
+ text.push(c);
+ } else {
+ text.extend(c.escape_default());
+ }
+ }
+ text.push('"');
+ Literal::_new(text)
+ }
+
+ pub fn character(t: char) -> Literal {
+ let mut text = String::new();
+ text.push('\'');
+ if t == '"' {
+ // escape_default turns this into '\"' which is unnecessary.
+ text.push(t);
+ } else {
+ text.extend(t.escape_default());
+ }
+ text.push('\'');
+ Literal::_new(text)
+ }
+
+ pub fn byte_string(bytes: &[u8]) -> Literal {
+ let mut escaped = "b\"".to_string();
+ for b in bytes {
+ match *b {
+ b'\0' => escaped.push_str(r"\0"),
+ b'\t' => escaped.push_str(r"\t"),
+ b'\n' => escaped.push_str(r"\n"),
+ b'\r' => escaped.push_str(r"\r"),
+ b'"' => escaped.push_str("\\\""),
+ b'\\' => escaped.push_str("\\\\"),
+ b'\x20'..=b'\x7E' => escaped.push(*b as char),
+ _ => escaped.push_str(&format!("\\x{:02X}", b)),
+ }
+ }
+ escaped.push('"');
+ Literal::_new(escaped)
+ }
+
+ pub fn span(&self) -> Span {
+ self.span
+ }
+
+ pub fn set_span(&mut self, span: Span) {
+ self.span = span;
+ }
+
+ pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> {
+ None
+ }
+}
+
+impl fmt::Display for Literal {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ self.text.fmt(f)
+ }
+}
+
+impl fmt::Debug for Literal {
+ fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+ let mut debug = fmt.debug_struct("Literal");
+ debug.field("lit", &format_args!("{}", self.text));
+ #[cfg(procmacro2_semver_exempt)]
+ debug.field("span", &self.span);
+ debug.finish()
+ }
+}
+
+fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
+ let mut trees = Vec::new();
+ loop {
+ let input_no_ws = skip_whitespace(input);
+ if input_no_ws.rest.len() == 0 {
+ break;
+ }
+ if let Ok((a, tokens)) = doc_comment(input_no_ws) {
+ input = a;
+ trees.extend(tokens);
+ continue;
+ }
+
+ let (a, tt) = match token_tree(input_no_ws) {
+ Ok(p) => p,
+ Err(_) => break,
+ };
+ trees.push(tt);
+ input = a;
+ }
+ Ok((input, TokenStream { inner: trees }))
+}
+
+#[cfg(not(span_locations))]
+fn spanned<'a, T>(
+ input: Cursor<'a>,
+ f: fn(Cursor<'a>) -> PResult<'a, T>,
+) -> PResult<'a, (T, crate::Span)> {
+ let (a, b) = f(skip_whitespace(input))?;
+ Ok((a, ((b, crate::Span::_new_stable(Span::call_site())))))
+}
+
+#[cfg(span_locations)]
+fn spanned<'a, T>(
+ input: Cursor<'a>,
+ f: fn(Cursor<'a>) -> PResult<'a, T>,
+) -> PResult<'a, (T, crate::Span)> {
+ let input = skip_whitespace(input);
+ let lo = input.off;
+ let (a, b) = f(input)?;
+ let hi = a.off;
+ let span = crate::Span::_new_stable(Span { lo, hi });
+ Ok((a, (b, span)))
+}
+
+fn token_tree(input: Cursor) -> PResult<TokenTree> {
+ let (rest, (mut tt, span)) = spanned(input, token_kind)?;
+ tt.set_span(span);
+ Ok((rest, tt))
+}
+
+named!(token_kind -> TokenTree, alt!(
+ map!(group, |g| TokenTree::Group(crate::Group::_new_stable(g)))
+ |
+ map!(literal, |l| TokenTree::Literal(crate::Literal::_new_stable(l))) // must be before symbol
+ |
+ map!(op, TokenTree::Punct)
+ |
+ symbol_leading_ws
+));
+
+named!(group -> Group, alt!(
+ delimited!(
+ punct!("("),
+ token_stream,
+ punct!(")")
+ ) => { |ts| Group::new(Delimiter::Parenthesis, ts) }
+ |
+ delimited!(
+ punct!("["),
+ token_stream,
+ punct!("]")
+ ) => { |ts| Group::new(Delimiter::Bracket, ts) }
+ |
+ delimited!(
+ punct!("{"),
+ token_stream,
+ punct!("}")
+ ) => { |ts| Group::new(Delimiter::Brace, ts) }
+));
+
+fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
+ symbol(skip_whitespace(input))
+}
+
+fn symbol(input: Cursor) -> PResult<TokenTree> {
+ let raw = input.starts_with("r#");
+ let rest = input.advance((raw as usize) << 1);
+
+ let (rest, sym) = symbol_not_raw(rest)?;
+
+ if !raw {
+ let ident = crate::Ident::new(sym, crate::Span::call_site());
+ return Ok((rest, ident.into()));
+ }
+
+ if sym == "_" {
+ return Err(LexError);
+ }
+
+ let ident = crate::Ident::_new_raw(sym, crate::Span::call_site());
+ Ok((rest, ident.into()))
+}
+
+fn symbol_not_raw(input: Cursor) -> PResult<&str> {
+ let mut chars = input.char_indices();
+
+ match chars.next() {
+ Some((_, ch)) if is_ident_start(ch) => {}
+ _ => return Err(LexError),
+ }
+
+ let mut end = input.len();
+ for (i, ch) in chars {
+ if !is_ident_continue(ch) {
+ end = i;
+ break;
+ }
+ }
+
+ Ok((input.advance(end), &input.rest[..end]))
+}
+
+fn literal(input: Cursor) -> PResult<Literal> {
+ let input_no_ws = skip_whitespace(input);
+
+ match literal_nocapture(input_no_ws) {
+ Ok((a, ())) => {
+ let start = input.len() - input_no_ws.len();
+ let len = input_no_ws.len() - a.len();
+ let end = start + len;
+ Ok((a, Literal::_new(input.rest[start..end].to_string())))
+ }
+ Err(LexError) => Err(LexError),
+ }
+}
+
+named!(literal_nocapture -> (), alt!(
+ string
+ |
+ byte_string
+ |
+ byte
+ |
+ character
+ |
+ float
+ |
+ int
+));
+
+named!(string -> (), alt!(
+ quoted_string
+ |
+ preceded!(
+ punct!("r"),
+ raw_string
+ ) => { |_| () }
+));
+
+named!(quoted_string -> (), do_parse!(
+ punct!("\"") >>
+ cooked_string >>
+ tag!("\"") >>
+ option!(symbol_not_raw) >>
+ (())
+));
+
+fn cooked_string(input: Cursor) -> PResult<()> {
+ let mut chars = input.char_indices().peekable();
+ while let Some((byte_offset, ch)) = chars.next() {
+ match ch {
+ '"' => {
+ return Ok((input.advance(byte_offset), ()));
+ }
+ '\r' => {
+ if let Some((_, '\n')) = chars.next() {
+ // ...
+ } else {
+ break;
+ }
+ }
+ '\\' => match chars.next() {
+ Some((_, 'x')) => {
+ if !backslash_x_char(&mut chars) {
+ break;
+ }
+ }
+ Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
+ | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
+ Some((_, 'u')) => {
+ if !backslash_u(&mut chars) {
+ break;
+ }
+ }
+ Some((_, '\n')) | Some((_, '\r')) => {
+ while let Some(&(_, ch)) = chars.peek() {
+ if ch.is_whitespace() {
+ chars.next();
+ } else {
+ break;
+ }
+ }
+ }
+ _ => break,
+ },
+ _ch => {}
+ }
+ }
+ Err(LexError)
+}
+
+named!(byte_string -> (), alt!(
+ delimited!(
+ punct!("b\""),
+ cooked_byte_string,
+ tag!("\"")
+ ) => { |_| () }
+ |
+ preceded!(
+ punct!("br"),
+ raw_string
+ ) => { |_| () }
+));
+
+fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
+ let mut bytes = input.bytes().enumerate();
+ 'outer: while let Some((offset, b)) = bytes.next() {
+ match b {
+ b'"' => {
+ return Ok((input.advance(offset), ()));
+ }
+ b'\r' => {
+ if let Some((_, b'\n')) = bytes.next() {
+ // ...
+ } else {
+ break;
+ }
+ }
+ b'\\' => match bytes.next() {
+ Some((_, b'x')) => {
+ if !backslash_x_byte(&mut bytes) {
+ break;
+ }
+ }
+ Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
+ | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
+ Some((newline, b'\n')) | Some((newline, b'\r')) => {
+ let rest = input.advance(newline + 1);
+ for (offset, ch) in rest.char_indices() {
+ if !ch.is_whitespace() {
+ input = rest.advance(offset);
+ bytes = input.bytes().enumerate();
+ continue 'outer;
+ }
+ }
+ break;
+ }
+ _ => break,
+ },
+ b if b < 0x80 => {}
+ _ => break,
+ }
+ }
+ Err(LexError)
+}
+
+fn raw_string(input: Cursor) -> PResult<()> {
+ let mut chars = input.char_indices();
+ let mut n = 0;
+ while let Some((byte_offset, ch)) = chars.next() {
+ match ch {
+ '"' => {
+ n = byte_offset;
+ break;
+ }
+ '#' => {}
+ _ => return Err(LexError),
+ }
+ }
+ for (byte_offset, ch) in chars {
+ match ch {
+ '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
+ let rest = input.advance(byte_offset + 1 + n);
+ return Ok((rest, ()));
+ }
+ '\r' => {}
+ _ => {}
+ }
+ }
+ Err(LexError)
+}
+
+named!(byte -> (), do_parse!(
+ punct!("b") >>
+ tag!("'") >>
+ cooked_byte >>
+ tag!("'") >>
+ (())
+));
+
+fn cooked_byte(input: Cursor) -> PResult<()> {
+ let mut bytes = input.bytes().enumerate();
+ let ok = match bytes.next().map(|(_, b)| b) {
+ Some(b'\\') => match bytes.next().map(|(_, b)| b) {
+ Some(b'x') => backslash_x_byte(&mut bytes),
+ Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
+ | Some(b'"') => true,
+ _ => false,
+ },
+ b => b.is_some(),
+ };
+ if ok {
+ match bytes.next() {
+ Some((offset, _)) => {
+ if input.chars().as_str().is_char_boundary(offset) {
+ Ok((input.advance(offset), ()))
+ } else {
+ Err(LexError)
+ }
+ }
+ None => Ok((input.advance(input.len()), ())),
+ }
+ } else {
+ Err(LexError)
+ }
+}
+
+named!(character -> (), do_parse!(
+ punct!("'") >>
+ cooked_char >>
+ tag!("'") >>
+ (())
+));
+
+fn cooked_char(input: Cursor) -> PResult<()> {
+ let mut chars = input.char_indices();
+ let ok = match chars.next().map(|(_, ch)| ch) {
+ Some('\\') => match chars.next().map(|(_, ch)| ch) {
+ Some('x') => backslash_x_char(&mut chars),
+ Some('u') => backslash_u(&mut chars),
+ Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
+ true
+ }
+ _ => false,
+ },
+ ch => ch.is_some(),
+ };
+ if ok {
+ match chars.next() {
+ Some((idx, _)) => Ok((input.advance(idx), ())),
+ None => Ok((input.advance(input.len()), ())),
+ }
+ } else {
+ Err(LexError)
+ }
+}
+
+macro_rules! next_ch {
+ ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
+ match $chars.next() {
+ Some((_, ch)) => match ch {
+ $pat $(| $rest)* => ch,
+ _ => return false,
+ },
+ None => return false
+ }
+ };
+}
+
+fn backslash_x_char<I>(chars: &mut I) -> bool
+where
+ I: Iterator<Item = (usize, char)>,
+{
+ next_ch!(chars @ '0'..='7');
+ next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
+ true
+}
+
+fn backslash_x_byte<I>(chars: &mut I) -> bool
+where
+ I: Iterator<Item = (usize, u8)>,
+{
+ next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
+ next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
+ true
+}
+
+fn backslash_u<I>(chars: &mut I) -> bool
+where
+ I: Iterator<Item = (usize, char)>,
+{
+ next_ch!(chars @ '{');
+ next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
+ loop {
+ let c = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F' | '_' | '}');
+ if c == '}' {
+ return true;
+ }
+ }
+}
+
+fn float(input: Cursor) -> PResult<()> {
+ let (mut rest, ()) = float_digits(input)?;
+ if let Some(ch) = rest.chars().next() {
+ if is_ident_start(ch) {
+ rest = symbol_not_raw(rest)?.0;
+ }
+ }
+ word_break(rest)
+}
+
+fn float_digits(input: Cursor) -> PResult<()> {
+ let mut chars = input.chars().peekable();
+ match chars.next() {
+ Some(ch) if ch >= '0' && ch <= '9' => {}
+ _ => return Err(LexError),
+ }
+
+ let mut len = 1;
+ let mut has_dot = false;
+ let mut has_exp = false;
+ while let Some(&ch) = chars.peek() {
+ match ch {
+ '0'..='9' | '_' => {
+ chars.next();
+ len += 1;
+ }
+ '.' => {
+ if has_dot {
+ break;
+ }
+ chars.next();
+ if chars
+ .peek()
+ .map(|&ch| ch == '.' || is_ident_start(ch))
+ .unwrap_or(false)
+ {
+ return Err(LexError);
+ }
+ len += 1;
+ has_dot = true;
+ }
+ 'e' | 'E' => {
+ chars.next();
+ len += 1;
+ has_exp = true;
+ break;
+ }
+ _ => break,
+ }
+ }
+
+ let rest = input.advance(len);
+ if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
+ return Err(LexError);
+ }
+
+ if has_exp {
+ let mut has_exp_value = false;
+ while let Some(&ch) = chars.peek() {
+ match ch {
+ '+' | '-' => {
+ if has_exp_value {
+ break;
+ }
+ chars.next();
+ len += 1;
+ }
+ '0'..='9' => {
+ chars.next();
+ len += 1;
+ has_exp_value = true;
+ }
+ '_' => {
+ chars.next();
+ len += 1;
+ }
+ _ => break,
+ }
+ }
+ if !has_exp_value {
+ return Err(LexError);
+ }
+ }
+
+ Ok((input.advance(len), ()))
+}
+
+fn int(input: Cursor) -> PResult<()> {
+ let (mut rest, ()) = digits(input)?;
+ if let Some(ch) = rest.chars().next() {
+ if is_ident_start(ch) {
+ rest = symbol_not_raw(rest)?.0;
+ }
+ }
+ word_break(rest)
+}
+
+fn digits(mut input: Cursor) -> PResult<()> {
+ let base = if input.starts_with("0x") {
+ input = input.advance(2);
+ 16
+ } else if input.starts_with("0o") {
+ input = input.advance(2);
+ 8
+ } else if input.starts_with("0b") {
+ input = input.advance(2);
+ 2
+ } else {
+ 10
+ };
+
+ let mut len = 0;
+ let mut empty = true;
+ for b in input.bytes() {
+ let digit = match b {
+ b'0'..=b'9' => (b - b'0') as u64,
+ b'a'..=b'f' => 10 + (b - b'a') as u64,
+ b'A'..=b'F' => 10 + (b - b'A') as u64,
+ b'_' => {
+ if empty && base == 10 {
+ return Err(LexError);
+ }
+ len += 1;
+ continue;
+ }
+ _ => break,
+ };
+ if digit >= base {
+ return Err(LexError);
+ }
+ len += 1;
+ empty = false;
+ }
+ if empty {
+ Err(LexError)
+ } else {
+ Ok((input.advance(len), ()))
+ }
+}
+
+fn op(input: Cursor) -> PResult<Punct> {
+ let input = skip_whitespace(input);
+ match op_char(input) {
+ Ok((rest, '\'')) => {
+ symbol(rest)?;
+ Ok((rest, Punct::new('\'', Spacing::Joint)))
+ }
+ Ok((rest, ch)) => {
+ let kind = match op_char(rest) {
+ Ok(_) => Spacing::Joint,
+ Err(LexError) => Spacing::Alone,
+ };
+ Ok((rest, Punct::new(ch, kind)))
+ }
+ Err(LexError) => Err(LexError),
+ }
+}
+
+fn op_char(input: Cursor) -> PResult<char> {
+ if input.starts_with("//") || input.starts_with("/*") {
+ // Do not accept `/` of a comment as an op.
+ return Err(LexError);
+ }
+
+ let mut chars = input.chars();
+ let first = match chars.next() {
+ Some(ch) => ch,
+ None => {
+ return Err(LexError);
+ }
+ };
+ let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
+ if recognized.contains(first) {
+ Ok((input.advance(first.len_utf8()), first))
+ } else {
+ Err(LexError)
+ }
+}
+
+fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
+ let mut trees = Vec::new();
+ let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
+ trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
+ if inner {
+ trees.push(Punct::new('!', Spacing::Alone).into());
+ }
+ let mut stream = vec![
+ TokenTree::Ident(crate::Ident::new("doc", span)),
+ TokenTree::Punct(Punct::new('=', Spacing::Alone)),
+ TokenTree::Literal(crate::Literal::string(comment)),
+ ];
+ for tt in stream.iter_mut() {
+ tt.set_span(span);
+ }
+ let group = Group::new(Delimiter::Bracket, stream.into_iter().collect());
+ trees.push(crate::Group::_new_stable(group).into());
+ for tt in trees.iter_mut() {
+ tt.set_span(span);
+ }
+ Ok((rest, trees))
+}
+
+named!(doc_comment_contents -> (&str, bool), alt!(
+ do_parse!(
+ punct!("//!") >>
+ s: take_until_newline_or_eof!() >>
+ ((s, true))
+ )
+ |
+ do_parse!(
+ option!(whitespace) >>
+ peek!(tag!("/*!")) >>
+ s: block_comment >>
+ ((s, true))
+ )
+ |
+ do_parse!(
+ punct!("///") >>
+ not!(tag!("/")) >>
+ s: take_until_newline_or_eof!() >>
+ ((s, false))
+ )
+ |
+ do_parse!(
+ option!(whitespace) >>
+ peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
+ s: block_comment >>
+ ((s, false))
+ )
+));
generated by cgit v1.2.1 (git 2.18.0) at 2024-05-19 00:02:24 +0000