From 5e20a29b4fdc8a2d442d1093681b396dcb4b816b Mon Sep 17 00:00:00 2001 From: Robin Krahl Date: Tue, 7 Jan 2020 11:18:04 +0000 Subject: Add structopt dependency in version 0.3.7 This patch series replaces argparse with structopt in the argument handling code. As a first step, we need structopt as a dependency. Import subrepo structopt/:structopt at efbdda4753592e27bc430fb01f7b9650b2f3174d Import subrepo bitflags/:bitflags at 30668016aca6bd3b02c766e8347e0b4080d4c296 Import subrepo clap/:clap at 784524f7eb193e35f81082cc69454c8c21b948f7 Import subrepo heck/:heck at 093d56fbf001e1506e56dbfa38631d99b1066df1 Import subrepo proc-macro-error/:proc-macro-error at 6c4cfe79a622c5de8ae68557993542be46eacae2 Import subrepo proc-macro2/:proc-macro2 at d5d48eddca4566e5438e8a2cbed4a74e049544de Import subrepo quote/:quote at 727436c6c137b20f0f34dde5d8fda2679b9747ad Import subrepo rustversion/:rustversion at 0c5663313516263059ce9059ef81fc7a1cf655ca Import subrepo syn-mid/:syn-mid at 5d3d85414a9e6674e1857ec22a87b96e04a6851a Import subrepo syn/:syn at e87c27e87f6f4ef8919d0372bdb056d53ef0d8f3 Import subrepo textwrap/:textwrap at abcd618beae3f74841032aa5b53c1086b0a57ca2 Import subrepo unicode-segmentation/:unicode-segmentation at 637c9874c4fe0c205ff27787faf150a40295c6c3 Import subrepo unicode-width/:unicode-width at 3033826f8bf05e82724140a981d5941e48fce393 Import subrepo unicode-xid/:unicode-xid at 4baae9fffb156ba229665b972a9cd5991787ceb7 --- proc-macro2/src/fallback.rs | 1458 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1458 insertions(+) create mode 100644 proc-macro2/src/fallback.rs (limited to 'proc-macro2/src/fallback.rs') diff --git a/proc-macro2/src/fallback.rs b/proc-macro2/src/fallback.rs new file mode 100644 index 0000000..9762d3b --- /dev/null +++ b/proc-macro2/src/fallback.rs @@ -0,0 +1,1458 @@ +#[cfg(span_locations)] +use std::cell::RefCell; +#[cfg(span_locations)] +use std::cmp; +use std::fmt; +use std::iter; +use std::ops::RangeBounds; +#[cfg(procmacro2_semver_exempt)] +use std::path::Path; +use std::path::PathBuf; +use std::str::FromStr; +use std::vec; + +use crate::strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult}; +use crate::{Delimiter, Punct, Spacing, TokenTree}; +use unicode_xid::UnicodeXID; + +#[derive(Clone)] +pub struct TokenStream { + inner: Vec, +} + +#[derive(Debug)] +pub struct LexError; + +impl TokenStream { + pub fn new() -> TokenStream { + TokenStream { inner: Vec::new() } + } + + pub fn is_empty(&self) -> bool { + self.inner.len() == 0 + } +} + +#[cfg(span_locations)] +fn get_cursor(src: &str) -> Cursor { + // Create a dummy file & add it to the source map + SOURCE_MAP.with(|cm| { + let mut cm = cm.borrow_mut(); + let name = format!("", cm.files.len()); + let span = cm.add_file(&name, src); + Cursor { + rest: src, + off: span.lo, + } + }) +} + +#[cfg(not(span_locations))] +fn get_cursor(src: &str) -> Cursor { + Cursor { rest: src } +} + +impl FromStr for TokenStream { + type Err = LexError; + + fn from_str(src: &str) -> Result { + // Create a dummy file & add it to the source map + let cursor = get_cursor(src); + + match token_stream(cursor) { + Ok((input, output)) => { + if skip_whitespace(input).len() != 0 { + Err(LexError) + } else { + Ok(output) + } + } + Err(LexError) => Err(LexError), + } + } +} + +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut joint = false; + for (i, tt) in self.inner.iter().enumerate() { + if i != 0 && !joint { + write!(f, " ")?; + } + joint = false; + match *tt { + TokenTree::Group(ref tt) => { + let (start, end) = match tt.delimiter() { + Delimiter::Parenthesis => ("(", ")"), + Delimiter::Brace => ("{", "}"), + Delimiter::Bracket => ("[", "]"), + Delimiter::None => ("", ""), + }; + if tt.stream().into_iter().next().is_none() { + write!(f, "{} {}", start, end)? + } else { + write!(f, "{} {} {}", start, tt.stream(), end)? + } + } + TokenTree::Ident(ref tt) => write!(f, "{}", tt)?, + TokenTree::Punct(ref tt) => { + write!(f, "{}", tt.as_char())?; + match tt.spacing() { + Spacing::Alone => {} + Spacing::Joint => joint = true, + } + } + TokenTree::Literal(ref tt) => write!(f, "{}", tt)?, + } + } + + Ok(()) + } +} + +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } +} + +#[cfg(use_proc_macro)] +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> TokenStream { + inner + .to_string() + .parse() + .expect("compiler token stream parse failed") + } +} + +#[cfg(use_proc_macro)] +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> proc_macro::TokenStream { + inner + .to_string() + .parse() + .expect("failed to parse to compiler tokens") + } +} + +impl From for TokenStream { + fn from(tree: TokenTree) -> TokenStream { + TokenStream { inner: vec![tree] } + } +} + +impl iter::FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut v = Vec::new(); + + for token in streams.into_iter() { + v.push(token); + } + + TokenStream { inner: v } + } +} + +impl iter::FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut v = Vec::new(); + + for stream in streams.into_iter() { + v.extend(stream.inner); + } + + TokenStream { inner: v } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner.extend(streams); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner + .extend(streams.into_iter().flat_map(|stream| stream)); + } +} + +pub type TokenTreeIter = vec::IntoIter; + +impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = TokenTreeIter; + + fn into_iter(self) -> TokenTreeIter { + self.inner.into_iter() + } +} + +#[derive(Clone, PartialEq, Eq)] +pub struct SourceFile { + path: PathBuf, +} + +impl SourceFile { + /// Get the path to this source file as a string. + pub fn path(&self) -> PathBuf { + self.path.clone() + } + + pub fn is_real(&self) -> bool { + // XXX(nika): Support real files in the future? + false + } +} + +impl fmt::Debug for SourceFile { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("SourceFile") + .field("path", &self.path()) + .field("is_real", &self.is_real()) + .finish() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LineColumn { + pub line: usize, + pub column: usize, +} + +#[cfg(span_locations)] +thread_local! { + static SOURCE_MAP: RefCell = RefCell::new(SourceMap { + // NOTE: We start with a single dummy file which all call_site() and + // def_site() spans reference. + files: vec![{ + #[cfg(procmacro2_semver_exempt)] + { + FileInfo { + name: "".to_owned(), + span: Span { lo: 0, hi: 0 }, + lines: vec![0], + } + } + + #[cfg(not(procmacro2_semver_exempt))] + { + FileInfo { + span: Span { lo: 0, hi: 0 }, + lines: vec![0], + } + } + }], + }); +} + +#[cfg(span_locations)] +struct FileInfo { + #[cfg(procmacro2_semver_exempt)] + name: String, + span: Span, + lines: Vec, +} + +#[cfg(span_locations)] +impl FileInfo { + fn offset_line_column(&self, offset: usize) -> LineColumn { + assert!(self.span_within(Span { + lo: offset as u32, + hi: offset as u32 + })); + let offset = offset - self.span.lo as usize; + match self.lines.binary_search(&offset) { + Ok(found) => LineColumn { + line: found + 1, + column: 0, + }, + Err(idx) => LineColumn { + line: idx, + column: offset - self.lines[idx - 1], + }, + } + } + + fn span_within(&self, span: Span) -> bool { + span.lo >= self.span.lo && span.hi <= self.span.hi + } +} + +/// Computesthe offsets of each line in the given source string. +#[cfg(span_locations)] +fn lines_offsets(s: &str) -> Vec { + let mut lines = vec![0]; + let mut prev = 0; + while let Some(len) = s[prev..].find('\n') { + prev += len + 1; + lines.push(prev); + } + lines +} + +#[cfg(span_locations)] +struct SourceMap { + files: Vec, +} + +#[cfg(span_locations)] +impl SourceMap { + fn next_start_pos(&self) -> u32 { + // Add 1 so there's always space between files. + // + // We'll always have at least 1 file, as we initialize our files list + // with a dummy file. + self.files.last().unwrap().span.hi + 1 + } + + fn add_file(&mut self, name: &str, src: &str) -> Span { + let lines = lines_offsets(src); + let lo = self.next_start_pos(); + // XXX(nika): Shouild we bother doing a checked cast or checked add here? + let span = Span { + lo, + hi: lo + (src.len() as u32), + }; + + #[cfg(procmacro2_semver_exempt)] + self.files.push(FileInfo { + name: name.to_owned(), + span, + lines, + }); + + #[cfg(not(procmacro2_semver_exempt))] + self.files.push(FileInfo { span, lines }); + let _ = name; + + span + } + + fn fileinfo(&self, span: Span) -> &FileInfo { + for file in &self.files { + if file.span_within(span) { + return file; + } + } + panic!("Invalid span with no related FileInfo!"); + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Span { + #[cfg(span_locations)] + lo: u32, + #[cfg(span_locations)] + hi: u32, +} + +impl Span { + #[cfg(not(span_locations))] + pub fn call_site() -> Span { + Span {} + } + + #[cfg(span_locations)] + pub fn call_site() -> Span { + Span { lo: 0, hi: 0 } + } + + #[cfg(procmacro2_semver_exempt)] + pub fn def_site() -> Span { + Span::call_site() + } + + #[cfg(procmacro2_semver_exempt)] + pub fn resolved_at(&self, _other: Span) -> Span { + // Stable spans consist only of line/column information, so + // `resolved_at` and `located_at` only select which span the + // caller wants line/column information from. + *self + } + + #[cfg(procmacro2_semver_exempt)] + pub fn located_at(&self, other: Span) -> Span { + other + } + + #[cfg(procmacro2_semver_exempt)] + pub fn source_file(&self) -> SourceFile { + SOURCE_MAP.with(|cm| { + let cm = cm.borrow(); + let fi = cm.fileinfo(*self); + SourceFile { + path: Path::new(&fi.name).to_owned(), + } + }) + } + + #[cfg(span_locations)] + pub fn start(&self) -> LineColumn { + SOURCE_MAP.with(|cm| { + let cm = cm.borrow(); + let fi = cm.fileinfo(*self); + fi.offset_line_column(self.lo as usize) + }) + } + + #[cfg(span_locations)] + pub fn end(&self) -> LineColumn { + SOURCE_MAP.with(|cm| { + let cm = cm.borrow(); + let fi = cm.fileinfo(*self); + fi.offset_line_column(self.hi as usize) + }) + } + + #[cfg(not(span_locations))] + pub fn join(&self, _other: Span) -> Option { + Some(Span {}) + } + + #[cfg(span_locations)] + pub fn join(&self, other: Span) -> Option { + SOURCE_MAP.with(|cm| { + let cm = cm.borrow(); + // If `other` is not within the same FileInfo as us, return None. + if !cm.fileinfo(*self).span_within(other) { + return None; + } + Some(Span { + lo: cmp::min(self.lo, other.lo), + hi: cmp::max(self.hi, other.hi), + }) + }) + } + + #[cfg(not(span_locations))] + fn first_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + fn first_byte(self) -> Self { + Span { + lo: self.lo, + hi: cmp::min(self.lo.saturating_add(1), self.hi), + } + } + + #[cfg(not(span_locations))] + fn last_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + fn last_byte(self) -> Self { + Span { + lo: cmp::max(self.hi.saturating_sub(1), self.lo), + hi: self.hi, + } + } +} + +impl fmt::Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + #[cfg(procmacro2_semver_exempt)] + return write!(f, "bytes({}..{})", self.lo, self.hi); + + #[cfg(not(procmacro2_semver_exempt))] + write!(f, "Span") + } +} + +pub fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { + if cfg!(procmacro2_semver_exempt) { + debug.field("span", &span); + } +} + +#[derive(Clone)] +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: Span, +} + +impl Group { + pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { + Group { + delimiter, + stream, + span: Span::call_site(), + } + } + + pub fn delimiter(&self) -> Delimiter { + self.delimiter + } + + pub fn stream(&self) -> TokenStream { + self.stream.clone() + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn span_open(&self) -> Span { + self.span.first_byte() + } + + pub fn span_close(&self) -> Span { + self.span.last_byte() + } + + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +impl fmt::Display for Group { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let (left, right) = match self.delimiter { + Delimiter::Parenthesis => ("(", ")"), + Delimiter::Brace => ("{", "}"), + Delimiter::Bracket => ("[", "]"), + Delimiter::None => ("", ""), + }; + + f.write_str(left)?; + self.stream.fmt(f)?; + f.write_str(right)?; + + Ok(()) + } +} + +impl fmt::Debug for Group { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Group"); + debug.field("delimiter", &self.delimiter); + debug.field("stream", &self.stream); + #[cfg(procmacro2_semver_exempt)] + debug.field("span", &self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub struct Ident { + sym: String, + span: Span, + raw: bool, +} + +impl Ident { + fn _new(string: &str, raw: bool, span: Span) -> Ident { + validate_ident(string); + + Ident { + sym: string.to_owned(), + span, + raw, + } + } + + pub fn new(string: &str, span: Span) -> Ident { + Ident::_new(string, false, span) + } + + pub fn new_raw(string: &str, span: Span) -> Ident { + Ident::_new(string, true, span) + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +fn is_ident_start(c: char) -> bool { + ('a' <= c && c <= 'z') + || ('A' <= c && c <= 'Z') + || c == '_' + || (c > '\x7f' && UnicodeXID::is_xid_start(c)) +} + +fn is_ident_continue(c: char) -> bool { + ('a' <= c && c <= 'z') + || ('A' <= c && c <= 'Z') + || c == '_' + || ('0' <= c && c <= '9') + || (c > '\x7f' && UnicodeXID::is_xid_continue(c)) +} + +fn validate_ident(string: &str) { + let validate = string; + if validate.is_empty() { + panic!("Ident is not allowed to be empty; use Option"); + } + + if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') { + panic!("Ident cannot be a number; use Literal instead"); + } + + fn ident_ok(string: &str) -> bool { + let mut chars = string.chars(); + let first = chars.next().unwrap(); + if !is_ident_start(first) { + return false; + } + for ch in chars { + if !is_ident_continue(ch) { + return false; + } + } + true + } + + if !ident_ok(validate) { + panic!("{:?} is not a valid Ident", string); + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + self.sym == other.sym && self.raw == other.raw + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + let other = other.as_ref(); + if self.raw { + other.starts_with("r#") && self.sym == other[2..] + } else { + self.sym == other + } + } +} + +impl fmt::Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.raw { + "r#".fmt(f)?; + } + self.sym.fmt(f) + } +} + +impl fmt::Debug for Ident { + // Ident(proc_macro), Ident(r#union) + #[cfg(not(procmacro2_semver_exempt))] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_tuple("Ident"); + debug.field(&format_args!("{}", self)); + debug.finish() + } + + // Ident { + // sym: proc_macro, + // span: bytes(128..138) + // } + #[cfg(procmacro2_semver_exempt)] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_struct("Ident"); + debug.field("sym", &format_args!("{}", self)); + debug.field("span", &self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub struct Literal { + text: String, + span: Span, +} + +macro_rules! suffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub fn $name(n: $kind) -> Literal { + Literal::_new(format!(concat!("{}", stringify!($kind)), n)) + } + )*) +} + +macro_rules! unsuffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub fn $name(n: $kind) -> Literal { + Literal::_new(n.to_string()) + } + )*) +} + +impl Literal { + fn _new(text: String) -> Literal { + Literal { + text, + span: Span::call_site(), + } + } + + suffixed_numbers! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + + f32_suffixed => f32, + f64_suffixed => f64, + } + + unsuffixed_numbers! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + pub fn f32_unsuffixed(f: f32) -> Literal { + let mut s = f.to_string(); + if !s.contains(".") { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub fn f64_unsuffixed(f: f64) -> Literal { + let mut s = f.to_string(); + if !s.contains(".") { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub fn string(t: &str) -> Literal { + let mut text = String::with_capacity(t.len() + 2); + text.push('"'); + for c in t.chars() { + if c == '\'' { + // escape_default turns this into "\'" which is unnecessary. + text.push(c); + } else { + text.extend(c.escape_default()); + } + } + text.push('"'); + Literal::_new(text) + } + + pub fn character(t: char) -> Literal { + let mut text = String::new(); + text.push('\''); + if t == '"' { + // escape_default turns this into '\"' which is unnecessary. + text.push(t); + } else { + text.extend(t.escape_default()); + } + text.push('\''); + Literal::_new(text) + } + + pub fn byte_string(bytes: &[u8]) -> Literal { + let mut escaped = "b\"".to_string(); + for b in bytes { + match *b { + b'\0' => escaped.push_str(r"\0"), + b'\t' => escaped.push_str(r"\t"), + b'\n' => escaped.push_str(r"\n"), + b'\r' => escaped.push_str(r"\r"), + b'"' => escaped.push_str("\\\""), + b'\\' => escaped.push_str("\\\\"), + b'\x20'..=b'\x7E' => escaped.push(*b as char), + _ => escaped.push_str(&format!("\\x{:02X}", b)), + } + } + escaped.push('"'); + Literal::_new(escaped) + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn set_span(&mut self, span: Span) { + self.span = span; + } + + pub fn subspan>(&self, _range: R) -> Option { + None + } +} + +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.text.fmt(f) + } +} + +impl fmt::Debug for Literal { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Literal"); + debug.field("lit", &format_args!("{}", self.text)); + #[cfg(procmacro2_semver_exempt)] + debug.field("span", &self.span); + debug.finish() + } +} + +fn token_stream(mut input: Cursor) -> PResult { + let mut trees = Vec::new(); + loop { + let input_no_ws = skip_whitespace(input); + if input_no_ws.rest.len() == 0 { + break; + } + if let Ok((a, tokens)) = doc_comment(input_no_ws) { + input = a; + trees.extend(tokens); + continue; + } + + let (a, tt) = match token_tree(input_no_ws) { + Ok(p) => p, + Err(_) => break, + }; + trees.push(tt); + input = a; + } + Ok((input, TokenStream { inner: trees })) +} + +#[cfg(not(span_locations))] +fn spanned<'a, T>( + input: Cursor<'a>, + f: fn(Cursor<'a>) -> PResult<'a, T>, +) -> PResult<'a, (T, crate::Span)> { + let (a, b) = f(skip_whitespace(input))?; + Ok((a, ((b, crate::Span::_new_stable(Span::call_site()))))) +} + +#[cfg(span_locations)] +fn spanned<'a, T>( + input: Cursor<'a>, + f: fn(Cursor<'a>) -> PResult<'a, T>, +) -> PResult<'a, (T, crate::Span)> { + let input = skip_whitespace(input); + let lo = input.off; + let (a, b) = f(input)?; + let hi = a.off; + let span = crate::Span::_new_stable(Span { lo, hi }); + Ok((a, (b, span))) +} + +fn token_tree(input: Cursor) -> PResult { + let (rest, (mut tt, span)) = spanned(input, token_kind)?; + tt.set_span(span); + Ok((rest, tt)) +} + +named!(token_kind -> TokenTree, alt!( + map!(group, |g| TokenTree::Group(crate::Group::_new_stable(g))) + | + map!(literal, |l| TokenTree::Literal(crate::Literal::_new_stable(l))) // must be before symbol + | + map!(op, TokenTree::Punct) + | + symbol_leading_ws +)); + +named!(group -> Group, alt!( + delimited!( + punct!("("), + token_stream, + punct!(")") + ) => { |ts| Group::new(Delimiter::Parenthesis, ts) } + | + delimited!( + punct!("["), + token_stream, + punct!("]") + ) => { |ts| Group::new(Delimiter::Bracket, ts) } + | + delimited!( + punct!("{"), + token_stream, + punct!("}") + ) => { |ts| Group::new(Delimiter::Brace, ts) } +)); + +fn symbol_leading_ws(input: Cursor) -> PResult { + symbol(skip_whitespace(input)) +} + +fn symbol(input: Cursor) -> PResult { + let raw = input.starts_with("r#"); + let rest = input.advance((raw as usize) << 1); + + let (rest, sym) = symbol_not_raw(rest)?; + + if !raw { + let ident = crate::Ident::new(sym, crate::Span::call_site()); + return Ok((rest, ident.into())); + } + + if sym == "_" { + return Err(LexError); + } + + let ident = crate::Ident::_new_raw(sym, crate::Span::call_site()); + Ok((rest, ident.into())) +} + +fn symbol_not_raw(input: Cursor) -> PResult<&str> { + let mut chars = input.char_indices(); + + match chars.next() { + Some((_, ch)) if is_ident_start(ch) => {} + _ => return Err(LexError), + } + + let mut end = input.len(); + for (i, ch) in chars { + if !is_ident_continue(ch) { + end = i; + break; + } + } + + Ok((input.advance(end), &input.rest[..end])) +} + +fn literal(input: Cursor) -> PResult { + let input_no_ws = skip_whitespace(input); + + match literal_nocapture(input_no_ws) { + Ok((a, ())) => { + let start = input.len() - input_no_ws.len(); + let len = input_no_ws.len() - a.len(); + let end = start + len; + Ok((a, Literal::_new(input.rest[start..end].to_string()))) + } + Err(LexError) => Err(LexError), + } +} + +named!(literal_nocapture -> (), alt!( + string + | + byte_string + | + byte + | + character + | + float + | + int +)); + +named!(string -> (), alt!( + quoted_string + | + preceded!( + punct!("r"), + raw_string + ) => { |_| () } +)); + +named!(quoted_string -> (), do_parse!( + punct!("\"") >> + cooked_string >> + tag!("\"") >> + option!(symbol_not_raw) >> + (()) +)); + +fn cooked_string(input: Cursor) -> PResult<()> { + let mut chars = input.char_indices().peekable(); + while let Some((byte_offset, ch)) = chars.next() { + match ch { + '"' => { + return Ok((input.advance(byte_offset), ())); + } + '\r' => { + if let Some((_, '\n')) = chars.next() { + // ... + } else { + break; + } + } + '\\' => match chars.next() { + Some((_, 'x')) => { + if !backslash_x_char(&mut chars) { + break; + } + } + Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\')) + | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {} + Some((_, 'u')) => { + if !backslash_u(&mut chars) { + break; + } + } + Some((_, '\n')) | Some((_, '\r')) => { + while let Some(&(_, ch)) = chars.peek() { + if ch.is_whitespace() { + chars.next(); + } else { + break; + } + } + } + _ => break, + }, + _ch => {} + } + } + Err(LexError) +} + +named!(byte_string -> (), alt!( + delimited!( + punct!("b\""), + cooked_byte_string, + tag!("\"") + ) => { |_| () } + | + preceded!( + punct!("br"), + raw_string + ) => { |_| () } +)); + +fn cooked_byte_string(mut input: Cursor) -> PResult<()> { + let mut bytes = input.bytes().enumerate(); + 'outer: while let Some((offset, b)) = bytes.next() { + match b { + b'"' => { + return Ok((input.advance(offset), ())); + } + b'\r' => { + if let Some((_, b'\n')) = bytes.next() { + // ... + } else { + break; + } + } + b'\\' => match bytes.next() { + Some((_, b'x')) => { + if !backslash_x_byte(&mut bytes) { + break; + } + } + Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\')) + | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {} + Some((newline, b'\n')) | Some((newline, b'\r')) => { + let rest = input.advance(newline + 1); + for (offset, ch) in rest.char_indices() { + if !ch.is_whitespace() { + input = rest.advance(offset); + bytes = input.bytes().enumerate(); + continue 'outer; + } + } + break; + } + _ => break, + }, + b if b < 0x80 => {} + _ => break, + } + } + Err(LexError) +} + +fn raw_string(input: Cursor) -> PResult<()> { + let mut chars = input.char_indices(); + let mut n = 0; + while let Some((byte_offset, ch)) = chars.next() { + match ch { + '"' => { + n = byte_offset; + break; + } + '#' => {} + _ => return Err(LexError), + } + } + for (byte_offset, ch) in chars { + match ch { + '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => { + let rest = input.advance(byte_offset + 1 + n); + return Ok((rest, ())); + } + '\r' => {} + _ => {} + } + } + Err(LexError) +} + +named!(byte -> (), do_parse!( + punct!("b") >> + tag!("'") >> + cooked_byte >> + tag!("'") >> + (()) +)); + +fn cooked_byte(input: Cursor) -> PResult<()> { + let mut bytes = input.bytes().enumerate(); + let ok = match bytes.next().map(|(_, b)| b) { + Some(b'\\') => match bytes.next().map(|(_, b)| b) { + Some(b'x') => backslash_x_byte(&mut bytes), + Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'') + | Some(b'"') => true, + _ => false, + }, + b => b.is_some(), + }; + if ok { + match bytes.next() { + Some((offset, _)) => { + if input.chars().as_str().is_char_boundary(offset) { + Ok((input.advance(offset), ())) + } else { + Err(LexError) + } + } + None => Ok((input.advance(input.len()), ())), + } + } else { + Err(LexError) + } +} + +named!(character -> (), do_parse!( + punct!("'") >> + cooked_char >> + tag!("'") >> + (()) +)); + +fn cooked_char(input: Cursor) -> PResult<()> { + let mut chars = input.char_indices(); + let ok = match chars.next().map(|(_, ch)| ch) { + Some('\\') => match chars.next().map(|(_, ch)| ch) { + Some('x') => backslash_x_char(&mut chars), + Some('u') => backslash_u(&mut chars), + Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => { + true + } + _ => false, + }, + ch => ch.is_some(), + }; + if ok { + match chars.next() { + Some((idx, _)) => Ok((input.advance(idx), ())), + None => Ok((input.advance(input.len()), ())), + } + } else { + Err(LexError) + } +} + +macro_rules! next_ch { + ($chars:ident @ $pat:pat $(| $rest:pat)*) => { + match $chars.next() { + Some((_, ch)) => match ch { + $pat $(| $rest)* => ch, + _ => return false, + }, + None => return false + } + }; +} + +fn backslash_x_char(chars: &mut I) -> bool +where + I: Iterator, +{ + next_ch!(chars @ '0'..='7'); + next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + true +} + +fn backslash_x_byte(chars: &mut I) -> bool +where + I: Iterator, +{ + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + true +} + +fn backslash_u(chars: &mut I) -> bool +where + I: Iterator, +{ + next_ch!(chars @ '{'); + next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + loop { + let c = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F' | '_' | '}'); + if c == '}' { + return true; + } + } +} + +fn float(input: Cursor) -> PResult<()> { + let (mut rest, ()) = float_digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = symbol_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn float_digits(input: Cursor) -> PResult<()> { + let mut chars = input.chars().peekable(); + match chars.next() { + Some(ch) if ch >= '0' && ch <= '9' => {} + _ => return Err(LexError), + } + + let mut len = 1; + let mut has_dot = false; + let mut has_exp = false; + while let Some(&ch) = chars.peek() { + match ch { + '0'..='9' | '_' => { + chars.next(); + len += 1; + } + '.' => { + if has_dot { + break; + } + chars.next(); + if chars + .peek() + .map(|&ch| ch == '.' || is_ident_start(ch)) + .unwrap_or(false) + { + return Err(LexError); + } + len += 1; + has_dot = true; + } + 'e' | 'E' => { + chars.next(); + len += 1; + has_exp = true; + break; + } + _ => break, + } + } + + let rest = input.advance(len); + if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) { + return Err(LexError); + } + + if has_exp { + let mut has_exp_value = false; + while let Some(&ch) = chars.peek() { + match ch { + '+' | '-' => { + if has_exp_value { + break; + } + chars.next(); + len += 1; + } + '0'..='9' => { + chars.next(); + len += 1; + has_exp_value = true; + } + '_' => { + chars.next(); + len += 1; + } + _ => break, + } + } + if !has_exp_value { + return Err(LexError); + } + } + + Ok((input.advance(len), ())) +} + +fn int(input: Cursor) -> PResult<()> { + let (mut rest, ()) = digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = symbol_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn digits(mut input: Cursor) -> PResult<()> { + let base = if input.starts_with("0x") { + input = input.advance(2); + 16 + } else if input.starts_with("0o") { + input = input.advance(2); + 8 + } else if input.starts_with("0b") { + input = input.advance(2); + 2 + } else { + 10 + }; + + let mut len = 0; + let mut empty = true; + for b in input.bytes() { + let digit = match b { + b'0'..=b'9' => (b - b'0') as u64, + b'a'..=b'f' => 10 + (b - b'a') as u64, + b'A'..=b'F' => 10 + (b - b'A') as u64, + b'_' => { + if empty && base == 10 { + return Err(LexError); + } + len += 1; + continue; + } + _ => break, + }; + if digit >= base { + return Err(LexError); + } + len += 1; + empty = false; + } + if empty { + Err(LexError) + } else { + Ok((input.advance(len), ())) + } +} + +fn op(input: Cursor) -> PResult { + let input = skip_whitespace(input); + match op_char(input) { + Ok((rest, '\'')) => { + symbol(rest)?; + Ok((rest, Punct::new('\'', Spacing::Joint))) + } + Ok((rest, ch)) => { + let kind = match op_char(rest) { + Ok(_) => Spacing::Joint, + Err(LexError) => Spacing::Alone, + }; + Ok((rest, Punct::new(ch, kind))) + } + Err(LexError) => Err(LexError), + } +} + +fn op_char(input: Cursor) -> PResult { + if input.starts_with("//") || input.starts_with("/*") { + // Do not accept `/` of a comment as an op. + return Err(LexError); + } + + let mut chars = input.chars(); + let first = match chars.next() { + Some(ch) => ch, + None => { + return Err(LexError); + } + }; + let recognized = "~!@#$%^&*-=+|;:,<.>/?'"; + if recognized.contains(first) { + Ok((input.advance(first.len_utf8()), first)) + } else { + Err(LexError) + } +} + +fn doc_comment(input: Cursor) -> PResult> { + let mut trees = Vec::new(); + let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?; + trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone))); + if inner { + trees.push(Punct::new('!', Spacing::Alone).into()); + } + let mut stream = vec![ + TokenTree::Ident(crate::Ident::new("doc", span)), + TokenTree::Punct(Punct::new('=', Spacing::Alone)), + TokenTree::Literal(crate::Literal::string(comment)), + ]; + for tt in stream.iter_mut() { + tt.set_span(span); + } + let group = Group::new(Delimiter::Bracket, stream.into_iter().collect()); + trees.push(crate::Group::_new_stable(group).into()); + for tt in trees.iter_mut() { + tt.set_span(span); + } + Ok((rest, trees)) +} + +named!(doc_comment_contents -> (&str, bool), alt!( + do_parse!( + punct!("//!") >> + s: take_until_newline_or_eof!() >> + ((s, true)) + ) + | + do_parse!( + option!(whitespace) >> + peek!(tag!("/*!")) >> + s: block_comment >> + ((s, true)) + ) + | + do_parse!( + punct!("///") >> + not!(tag!("/")) >> + s: take_until_newline_or_eof!() >> + ((s, false)) + ) + | + do_parse!( + option!(whitespace) >> + peek!(tuple!(tag!("/**"), not!(tag!("*")))) >> + s: block_comment >> + ((s, false)) + ) +)); -- cgit v1.2.1