diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 4797ac7c66ac6..ff50636a8d982 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -163,7 +163,8 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader, token::Lifetime(..) => "lifetime", token::DocComment(..) => "doccomment", - token::Underscore | token::Eof | token::Interpolated(..) => "", + token::Underscore | token::Eof | token::Interpolated(..) | + token::MatchNt(..) | token::SubstNt(..) => "", }; // as mentioned above, use the original source code instead of diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 6a354fa20e1b4..20f7caac48222 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -10,7 +10,7 @@ // The Rust abstract syntax tree. -use codemap::{Span, Spanned, DUMMY_SP, ExpnId}; +use codemap::{Span, Spanned, DUMMY_SP, ExpnId, respan}; use abi::Abi; use ast_util; use owned_slice::OwnedSlice; @@ -713,6 +713,19 @@ impl Delimited { } } +/// A sequence of token treesee +#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] +pub struct SequenceRepetition { + /// The sequence of token trees + pub tts: Vec, + /// The optional separator + pub separator: Option, + /// Whether the sequence can be repeated zero (*), or one or more times (+) + pub op: KleeneOp, + /// The number of `MatchNt`s that appear in the sequence (and subsequences) + pub num_captures: uint, +} + /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) /// for token sequences. #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] @@ -727,14 +740,12 @@ pub enum KleeneOp { /// be passed to syntax extensions using a uniform type. /// /// If the syntax extension is an MBE macro, it will attempt to match its -/// LHS "matchers" against the provided token tree, and if it finds a +/// LHS token tree against the provided token tree, and if it finds a /// match, will transcribe the RHS token tree, splicing in any captured -/// `macro_parser::matched_nonterminals` into the `TtNonterminal`s it finds. +/// macro_parser::matched_nonterminals into the `SubstNt`s it finds. /// -/// The RHS of an MBE macro is the only place a `TtNonterminal` or `TtSequence` -/// makes any real sense. You could write them elsewhere but nothing -/// else knows what to do with them, so you'll probably get a syntax -/// error. +/// The RHS of an MBE macro is the only place `SubstNt`s are substituted. +/// Nothing special happens to misnamed or misplaced `SubstNt`s. #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] #[doc="For macro invocations; parsing is delegated to the macro"] pub enum TokenTree { @@ -743,90 +754,82 @@ pub enum TokenTree { /// A delimited sequence of token trees TtDelimited(Span, Rc), - // These only make sense for right-hand-sides of MBE macros: + // This only makes sense in MBE macros. - /// A Kleene-style repetition sequence with an optional separator. - // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TtSequence(Span, Rc>, Option, KleeneOp), - /// A syntactic variable that will be filled in by macro expansion. - TtNonterminal(Span, Ident) + /// A kleene-style repetition sequence with a span + // FIXME(eddyb) #12938 Use DST. + TtSequence(Span, Rc), } impl TokenTree { + pub fn len(&self) -> uint { + match *self { + TtToken(_, token::DocComment(_)) => 2, + TtToken(_, token::SubstNt(..)) => 2, + TtToken(_, token::MatchNt(..)) => 3, + TtDelimited(_, ref delimed) => { + delimed.tts.len() + 2 + } + TtSequence(_, ref seq) => { + seq.tts.len() + } + TtToken(..) => 0 + } + } + + pub fn get_tt(&self, index: uint) -> TokenTree { + match (self, index) { + (&TtToken(sp, token::DocComment(_)), 0) => { + TtToken(sp, token::Pound) + } + (&TtToken(sp, token::DocComment(name)), 1) => { + let doc = MetaNameValue(token::intern_and_get_ident("doc"), + respan(sp, LitStr(token::get_name(name), CookedStr))); + let doc = token::NtMeta(P(respan(sp, doc))); + TtDelimited(sp, Rc::new(Delimited { + delim: token::Bracket, + open_span: sp, + tts: vec![TtToken(sp, token::Interpolated(doc))], + close_span: sp, + })) + } + (&TtDelimited(_, ref delimed), _) => { + if index == 0 { + return delimed.open_tt(); + } + if index == delimed.tts.len() + 1 { + return delimed.close_tt(); + } + delimed.tts[index - 1].clone() + } + (&TtToken(sp, token::SubstNt(name, name_st)), _) => { + let v = [TtToken(sp, token::Dollar), + TtToken(sp, token::Ident(name, name_st))]; + v[index] + } + (&TtToken(sp, token::MatchNt(name, kind, name_st, kind_st)), _) => { + let v = [TtToken(sp, token::SubstNt(name, name_st)), + TtToken(sp, token::Colon), + TtToken(sp, token::Ident(kind, kind_st))]; + v[index] + } + (&TtSequence(_, ref seq), _) => { + seq.tts[index].clone() + } + _ => panic!("Cannot expand a token tree") + } + } + /// Returns the `Span` corresponding to this token tree. pub fn get_span(&self) -> Span { match *self { - TtToken(span, _) => span, - TtDelimited(span, _) => span, - TtSequence(span, _, _, _) => span, - TtNonterminal(span, _) => span, + TtToken(span, _) => span, + TtDelimited(span, _) => span, + TtSequence(span, _) => span, } } } -// Matchers are nodes defined-by and recognized-by the main rust parser and -// language, but they're only ever found inside syntax-extension invocations; -// indeed, the only thing that ever _activates_ the rules in the rust parser -// for parsing a matcher is a matcher looking for the 'matchers' nonterminal -// itself. Matchers represent a small sub-language for pattern-matching -// token-trees, and are thus primarily used by the macro-defining extension -// itself. -// -// MatchTok -// -------- -// -// A matcher that matches a single token, denoted by the token itself. So -// long as there's no $ involved. -// -// -// MatchSeq -// -------- -// -// A matcher that matches a sequence of sub-matchers, denoted various -// possible ways: -// -// $(M)* zero or more Ms -// $(M)+ one or more Ms -// $(M),+ one or more comma-separated Ms -// $(A B C);* zero or more semi-separated 'A B C' seqs -// -// -// MatchNonterminal -// ----------------- -// -// A matcher that matches one of a few interesting named rust -// nonterminals, such as types, expressions, items, or raw token-trees. A -// black-box matcher on expr, for example, binds an expr to a given ident, -// and that ident can re-occur as an interpolation in the RHS of a -// macro-by-example rule. For example: -// -// $foo:expr => 1 + $foo // interpolate an expr -// $foo:tt => $foo // interpolate a token-tree -// $foo:tt => bar! $foo // only other valid interpolation -// // is in arg position for another -// // macro -// -// As a final, horrifying aside, note that macro-by-example's input is -// also matched by one of these matchers. Holy self-referential! It is matched -// by a MatchSeq, specifically this one: -// -// $( $lhs:matchers => $rhs:tt );+ -// -// If you understand that, you have closed the loop and understand the whole -// macro system. Congratulations. -pub type Matcher = Spanned; - -#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] -pub enum Matcher_ { - /// Match one token - MatchTok(token::Token), - /// Match repetitions of a sequence: body, separator, Kleene operator, - /// lo, hi position-in-match-array used: - MatchSeq(Vec, Option, KleeneOp, uint, uint), - /// Parse a Rust NT: name to bind, name of NT, position in match array: - MatchNonterminal(Ident, Ident, uint) -} - pub type Mac = Spanned; /// Represents a macro invocation. The Path indicates which macro diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index f751655c9ff62..db6be89e6e9ef 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -616,6 +616,20 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { vec!(mk_name(cx, sp, ident.ident()))); } + token::MatchNt(name, kind, name_style, kind_style) => { + return cx.expr_call(sp, + mk_token_path(cx, sp, "MatchNt"), + vec![mk_ident(cx, sp, name), + mk_ident(cx, sp, kind), + match name_style { + ModName => mk_token_path(cx, sp, "ModName"), + Plain => mk_token_path(cx, sp, "Plain"), + }, + match kind_style { + ModName => mk_token_path(cx, sp, "ModName"), + Plain => mk_token_path(cx, sp, "Plain"), + }]); + } token::Interpolated(_) => panic!("quote! with interpolated token"), _ => () @@ -654,6 +668,25 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { match *tt { + ast::TtToken(sp, SubstNt(ident, _)) => { + // tt.extend($ident.to_tokens(ext_cx).into_iter()) + + let e_to_toks = + cx.expr_method_call(sp, + cx.expr_ident(sp, ident), + id_ext("to_tokens"), + vec!(cx.expr_ident(sp, id_ext("ext_cx")))); + let e_to_toks = + cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]); + + let e_push = + cx.expr_method_call(sp, + cx.expr_ident(sp, id_ext("tt")), + id_ext("extend"), + vec!(e_to_toks)); + + vec!(cx.stmt_expr(e_push)) + } ast::TtToken(sp, ref tok) => { let e_sp = cx.expr_ident(sp, id_ext("_sp")); let e_tok = cx.expr_call(sp, @@ -673,25 +706,6 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { .collect() }, ast::TtSequence(..) => panic!("TtSequence in quote!"), - ast::TtNonterminal(sp, ident) => { - // tt.extend($ident.to_tokens(ext_cx).into_iter()) - - let e_to_toks = - cx.expr_method_call(sp, - cx.expr_ident(sp, ident), - id_ext("to_tokens"), - vec!(cx.expr_ident(sp, id_ext("ext_cx")))); - let e_to_toks = - cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]); - - let e_push = - cx.expr_method_call(sp, - cx.expr_ident(sp, id_ext("tt")), - id_ext("extend"), - vec!(e_to_toks)); - - vec!(cx.stmt_expr(e_push)) - }, } } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index b522d5881d762..1f0b667259473 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -78,126 +78,165 @@ use ast; -use ast::{Matcher, MatchTok, MatchSeq, MatchNonterminal, Ident}; +use ast::{TokenTree, Ident}; +use ast::{TtDelimited, TtSequence, TtToken}; use codemap::{BytePos, mk_sp}; use codemap; use parse::lexer::*; //resolve bug? use parse::ParseSess; use parse::attr::ParserAttr; use parse::parser::{LifetimeAndTypesWithoutColons, Parser}; +use parse::token::{Eof, DocComment, MatchNt, SubstNt}; use parse::token::{Token, Nonterminal}; use parse::token; use print::pprust; use ptr::P; +use std::mem; use std::rc::Rc; use std::collections::HashMap; +use std::collections::hash_map::{Vacant, Occupied}; -/* to avoid costly uniqueness checks, we require that `MatchSeq` always has a -nonempty body. */ +// To avoid costly uniqueness checks, we require that `MatchSeq` always has +// a nonempty body. +#[deriving(Clone)] +enum TokenTreeOrTokenTreeVec { + Tt(ast::TokenTree), + TtSeq(Rc>), +} + +impl TokenTreeOrTokenTreeVec { + fn len(&self) -> uint { + match self { + &TtSeq(ref v) => v.len(), + &Tt(ref tt) => tt.len(), + } + } + + fn get_tt(&self, index: uint) -> TokenTree { + match self { + &TtSeq(ref v) => v[index].clone(), + &Tt(ref tt) => tt.get_tt(index), + } + } +} + +/// an unzipping of `TokenTree`s +#[deriving(Clone)] +struct MatcherTtFrame { + elts: TokenTreeOrTokenTreeVec, + idx: uint, +} #[deriving(Clone)] pub struct MatcherPos { - elts: Vec , // maybe should be <'>? Need to understand regions. + stack: Vec, + top_elts: TokenTreeOrTokenTreeVec, sep: Option, idx: uint, up: Option>, matches: Vec>>, - match_lo: uint, match_hi: uint, + match_lo: uint, + match_cur: uint, + match_hi: uint, sp_lo: BytePos, } -pub fn count_names(ms: &[Matcher]) -> uint { - ms.iter().fold(0, |ct, m| { - ct + match m.node { - MatchTok(_) => 0u, - MatchSeq(ref more_ms, _, _, _, _) => { - count_names(more_ms.as_slice()) +pub fn count_names(ms: &[TokenTree]) -> uint { + ms.iter().fold(0, |count, elt| { + count + match elt { + &TtSequence(_, ref seq) => { + seq.num_captures } - MatchNonterminal(_, _, _) => 1u - }}) -} - -pub fn initial_matcher_pos(ms: Vec , sep: Option, lo: BytePos) - -> Box { - let mut match_idx_hi = 0u; - for elt in ms.iter() { - match elt.node { - MatchTok(_) => (), - MatchSeq(_,_,_,_,hi) => { - match_idx_hi = hi; // it is monotonic... + &TtDelimited(_, ref delim) => { + count_names(delim.tts.as_slice()) } - MatchNonterminal(_,_,pos) => { - match_idx_hi = pos+1u; // ...so latest is highest + &TtToken(_, MatchNt(..)) => { + 1 } + &TtToken(_, _) => 0, } - } - let matches = Vec::from_fn(count_names(ms.as_slice()), |_i| Vec::new()); + }) +} + +pub fn initial_matcher_pos(ms: Rc>, sep: Option, lo: BytePos) + -> Box { + let match_idx_hi = count_names(ms.as_slice()); + let matches = Vec::from_fn(match_idx_hi, |_i| Vec::new()); box MatcherPos { - elts: ms, + stack: vec![], + top_elts: TtSeq(ms), sep: sep, idx: 0u, up: None, matches: matches, match_lo: 0u, + match_cur: 0u, match_hi: match_idx_hi, sp_lo: lo } } -/// NamedMatch is a pattern-match result for a single ast::MatchNonterminal: +/// NamedMatch is a pattern-match result for a single token::MATCH_NONTERMINAL: /// so it is associated with a single ident in a parse, and all -/// MatchedNonterminal's in the NamedMatch have the same nonterminal type -/// (expr, item, etc). All the leaves in a single NamedMatch correspond to a -/// single matcher_nonterminal in the ast::Matcher that produced it. +/// `MatchedNonterminal`s in the NamedMatch have the same nonterminal type +/// (expr, item, etc). Each leaf in a single NamedMatch corresponds to a +/// single token::MATCH_NONTERMINAL in the TokenTree that produced it. /// -/// It should probably be renamed, it has more or less exact correspondence to -/// ast::match nodes, and the in-memory structure of a particular NamedMatch -/// represents the match that occurred when a particular subset of an -/// ast::match -- those ast::Matcher nodes leading to a single -/// MatchNonterminal -- was applied to a particular token tree. +/// The in-memory structure of a particular NamedMatch represents the match +/// that occurred when a particular subset of a matcher was applied to a +/// particular token tree. /// /// The width of each MatchedSeq in the NamedMatch, and the identity of the -/// MatchedNonterminal's, will depend on the token tree it was applied to: each -/// MatchedSeq corresponds to a single MatchSeq in the originating -/// ast::Matcher. The depth of the NamedMatch structure will therefore depend -/// only on the nesting depth of ast::MatchSeq's in the originating -/// ast::Matcher it was derived from. +/// `MatchedNonterminal`s, will depend on the token tree it was applied to: +/// each MatchedSeq corresponds to a single TTSeq in the originating +/// token tree. The depth of the NamedMatch structure will therefore depend +/// only on the nesting depth of `ast::TTSeq`s in the originating +/// token tree it was derived from. pub enum NamedMatch { MatchedSeq(Vec>, codemap::Span), MatchedNonterminal(Nonterminal) } -pub fn nameize(p_s: &ParseSess, ms: &[Matcher], res: &[Rc]) +pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc]) -> HashMap> { - fn n_rec(p_s: &ParseSess, m: &Matcher, res: &[Rc], - ret_val: &mut HashMap>) { - match *m { - codemap::Spanned {node: MatchTok(_), .. } => (), - codemap::Spanned {node: MatchSeq(ref more_ms, _, _, _, _), .. } => { - for next_m in more_ms.iter() { - n_rec(p_s, next_m, res, ret_val) - }; - } - codemap::Spanned { - node: MatchNonterminal(bind_name, _, idx), - span - } => { - if ret_val.contains_key(&bind_name) { - let string = token::get_ident(bind_name); - p_s.span_diagnostic - .span_fatal(span, - format!("duplicated bind name: {}", - string.get()).as_slice()) + fn n_rec(p_s: &ParseSess, m: &TokenTree, res: &[Rc], + ret_val: &mut HashMap>, idx: &mut uint) { + match m { + &TtSequence(_, ref seq) => { + for next_m in seq.tts.iter() { + n_rec(p_s, next_m, res, ret_val, idx) + } + } + &TtDelimited(_, ref delim) => { + for next_m in delim.tts.iter() { + n_rec(p_s, next_m, res, ret_val, idx) + } } - ret_val.insert(bind_name, res[idx].clone()); - } + &TtToken(sp, MatchNt(bind_name, _, _, _)) => { + match ret_val.entry(bind_name) { + Vacant(spot) => { + spot.set(res[*idx].clone()); + *idx += 1; + } + Occupied(..) => { + let string = token::get_ident(bind_name); + p_s.span_diagnostic + .span_fatal(sp, + format!("duplicated bind name: {}", + string.get()).as_slice()) + } + } + } + &TtToken(_, SubstNt(..)) => panic!("Cannot fill in a NT"), + &TtToken(_, _) => (), } } let mut ret_val = HashMap::new(); - for m in ms.iter() { n_rec(p_s, m, res, &mut ret_val) } + let mut idx = 0u; + for m in ms.iter() { n_rec(p_s, m, res, &mut ret_val, &mut idx) } ret_val } @@ -210,7 +249,7 @@ pub enum ParseResult { pub fn parse_or_else(sess: &ParseSess, cfg: ast::CrateConfig, rdr: TtReader, - ms: Vec ) + ms: Vec ) -> HashMap> { match parse(sess, cfg, rdr, ms.as_slice()) { Success(m) => m, @@ -237,12 +276,12 @@ pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { pub fn parse(sess: &ParseSess, cfg: ast::CrateConfig, mut rdr: TtReader, - ms: &[Matcher]) + ms: &[TokenTree]) -> ParseResult { let mut cur_eis = Vec::new(); - cur_eis.push(initial_matcher_pos(ms.iter() - .map(|x| (*x).clone()) - .collect(), + cur_eis.push(initial_matcher_pos(Rc::new(ms.iter() + .map(|x| (*x).clone()) + .collect()), None, rdr.peek().sp.lo)); @@ -255,13 +294,24 @@ pub fn parse(sess: &ParseSess, /* we append new items to this while we go */ loop { - let ei = match cur_eis.pop() { + let mut ei = match cur_eis.pop() { None => break, /* for each Earley Item */ Some(ei) => ei, }; + // When unzipped trees end, remove them + while ei.idx >= ei.top_elts.len() { + match ei.stack.pop() { + Some(MatcherTtFrame { elts, idx }) => { + ei.top_elts = elts; + ei.idx = idx + 1; + } + None => break + } + } + let idx = ei.idx; - let len = ei.elts.len(); + let len = ei.top_elts.len(); /* at end of sequence */ if idx >= len { @@ -293,6 +343,7 @@ pub fn parse(sess: &ParseSess, sp.hi)))); } + new_pos.match_cur = ei.match_hi; new_pos.idx += 1; cur_eis.push(new_pos); } @@ -301,69 +352,86 @@ pub fn parse(sess: &ParseSess, // the *_t vars are workarounds for the lack of unary move match ei.sep { - Some(ref t) if idx == len => { // we need a separator - // i'm conflicted about whether this should be hygienic.... - // though in this case, if the separators are never legal - // idents, it shouldn't matter. - if token_name_eq(&tok, t) { //pass the separator - let mut ei_t = ei.clone(); - ei_t.idx += 1; - next_eis.push(ei_t); + Some(ref t) if idx == len => { // we need a separator + // i'm conflicted about whether this should be hygienic.... + // though in this case, if the separators are never legal + // idents, it shouldn't matter. + if token_name_eq(&tok, t) { //pass the separator + let mut ei_t = ei.clone(); + // ei_t.match_cur = ei_t.match_lo; + ei_t.idx += 1; + next_eis.push(ei_t); + } + } + _ => { // we don't need a separator + let mut ei_t = ei; + ei_t.match_cur = ei_t.match_lo; + ei_t.idx = 0; + cur_eis.push(ei_t); } - } - _ => { // we don't need a separator - let mut ei_t = ei; - ei_t.idx = 0; - cur_eis.push(ei_t); - } } } else { eof_eis.push(ei); } } else { - match ei.elts[idx].node.clone() { - /* need to descend into sequence */ - MatchSeq(ref matchers, ref sep, kleene_op, - match_idx_lo, match_idx_hi) => { - if kleene_op == ast::ZeroOrMore { - let mut new_ei = ei.clone(); - new_ei.idx += 1u; - //we specifically matched zero repeats. - for idx in range(match_idx_lo, match_idx_hi) { - new_ei.matches[idx] - .push(Rc::new(MatchedSeq(Vec::new(), sp))); + match ei.top_elts.get_tt(idx) { + /* need to descend into sequence */ + TtSequence(sp, seq) => { + if seq.op == ast::ZeroOrMore { + let mut new_ei = ei.clone(); + new_ei.match_cur += seq.num_captures; + new_ei.idx += 1u; + //we specifically matched zero repeats. + for idx in range(ei.match_cur, ei.match_cur + seq.num_captures) { + new_ei.matches[idx].push(Rc::new(MatchedSeq(Vec::new(), sp))); + } + + cur_eis.push(new_ei); } - cur_eis.push(new_ei); + let matches = Vec::from_elem(ei.matches.len(), Vec::new()); + let ei_t = ei; + cur_eis.push(box MatcherPos { + stack: vec![], + sep: seq.separator.clone(), + idx: 0u, + matches: matches, + match_lo: ei_t.match_cur, + match_cur: ei_t.match_cur, + match_hi: ei_t.match_cur + seq.num_captures, + up: Some(ei_t), + sp_lo: sp.lo, + top_elts: Tt(TtSequence(sp, seq)), + }); } - - let matches = Vec::from_elem(ei.matches.len(), Vec::new()); - let ei_t = ei; - cur_eis.push(box MatcherPos { - elts: (*matchers).clone(), - sep: (*sep).clone(), - idx: 0u, - up: Some(ei_t), - matches: matches, - match_lo: match_idx_lo, match_hi: match_idx_hi, - sp_lo: sp.lo - }); - } - MatchNonterminal(_,_,_) => { - // Built-in nonterminals never start with these tokens, - // so we can eliminate them from consideration. - match tok { - token::CloseDelim(_) => {}, - _ => bb_eis.push(ei), + TtToken(_, MatchNt(..)) => { + // Built-in nonterminals never start with these tokens, + // so we can eliminate them from consideration. + match tok { + token::CloseDelim(_) => {}, + _ => bb_eis.push(ei), + } } - } - MatchTok(ref t) => { - let mut ei_t = ei.clone(); - if token_name_eq(t,&tok) { - ei_t.idx += 1; - next_eis.push(ei_t); + TtToken(sp, SubstNt(..)) => { + return Error(sp, "Cannot transcribe in macro LHS".into_string()) + } + seq @ TtDelimited(..) | seq @ TtToken(_, DocComment(..)) => { + let lower_elts = mem::replace(&mut ei.top_elts, Tt(seq)); + let idx = ei.idx; + ei.stack.push(MatcherTtFrame { + elts: lower_elts, + idx: idx, + }); + ei.idx = 0; + cur_eis.push(ei); + } + TtToken(_, ref t) => { + let mut ei_t = ei.clone(); + if token_name_eq(t,&tok) { + ei_t.idx += 1; + next_eis.push(ei_t); + } } - } } } } @@ -385,8 +453,8 @@ pub fn parse(sess: &ParseSess, if (bb_eis.len() > 0u && next_eis.len() > 0u) || bb_eis.len() > 1u { let nts = bb_eis.iter().map(|ei| { - match ei.elts[ei.idx].node { - MatchNonterminal(bind, name, _) => { + match ei.top_elts.get_tt(ei.idx) { + TtToken(_, MatchNt(bind, name, _, _)) => { (format!("{} ('{}')", token::get_ident(name), token::get_ident(bind))).to_string() @@ -410,12 +478,14 @@ pub fn parse(sess: &ParseSess, let mut rust_parser = Parser::new(sess, cfg.clone(), box rdr.clone()); let mut ei = bb_eis.pop().unwrap(); - match ei.elts[ei.idx].node { - MatchNonterminal(_, name, idx) => { + match ei.top_elts.get_tt(ei.idx) { + TtToken(_, MatchNt(_, name, _, _)) => { let name_string = token::get_ident(name); - ei.matches[idx].push(Rc::new(MatchedNonterminal( + let match_cur = ei.match_cur; + ei.matches[match_cur].push(Rc::new(MatchedNonterminal( parse_nt(&mut rust_parser, name_string.get())))); ei.idx += 1u; + ei.match_cur += 1; } _ => panic!() } @@ -461,7 +531,6 @@ pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal { p.quote_depth -= 1u; res } - "matchers" => token::NtMatchers(p.parse_matchers()), _ => { p.fatal(format!("unsupported builtin nonterminal parser: {}", name).as_slice()) diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index e50d4457af249..92c68b7a9c724 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -8,9 +8,9 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TtDelimited}; +use ast::{Ident, TtDelimited, TtSequence, TtToken}; use ast; -use codemap::{Span, Spanned, DUMMY_SP}; +use codemap::{Span, DUMMY_SP}; use ext::base::{ExtCtxt, MacResult, MacroDef}; use ext::base::{NormalTT, TTMacroExpander}; use ext::tt::macro_parser::{Success, Error, Failure}; @@ -20,7 +20,7 @@ use parse::lexer::new_tt_reader; use parse::parser::Parser; use parse::attr::ParserAttr; use parse::token::{special_idents, gensym_ident}; -use parse::token::{NtMatchers, NtTT}; +use parse::token::{MatchNt, NtTT}; use parse::token; use print; use ptr::P; @@ -158,14 +158,19 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers match **lhs { - MatchedNonterminal(NtMatchers(ref mtcs)) => { + MatchedNonterminal(NtTT(ref lhs_tt)) => { + let lhs_tt = match **lhs_tt { + TtDelimited(_, ref delim) => delim.tts.as_slice(), + _ => cx.span_fatal(sp, "malformed macro lhs") + }; // `None` is because we're not interpolating - let arg_rdr = new_tt_reader(&cx.parse_sess().span_diagnostic, - None, - arg.iter() - .map(|x| (*x).clone()) - .collect()); - match parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtcs.as_slice()) { + let mut arg_rdr = new_tt_reader(&cx.parse_sess().span_diagnostic, + None, + arg.iter() + .map(|x| (*x).clone()) + .collect()); + arg_rdr.desugar_doc_comments = true; + match parse(cx.parse_sess(), cx.cfg(), arg_rdr, lhs_tt) { Success(named_matches) => { let rhs = match *rhses[i] { // okay, what's your transcriber? @@ -202,6 +207,11 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, cx.span_fatal(best_fail_spot, best_fail_msg.as_slice()); } +// Note that macro-by-example's input is also matched against a token tree: +// $( $lhs:tt => $rhs:tt );+ +// +// Holy self-referential! + /// This procedure performs the expansion of the /// macro_rules! macro. It parses the RHS and adds /// an extension to the current context. @@ -210,31 +220,37 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt, name: Ident, arg: Vec ) -> Box { - // these spans won't matter, anyways - fn ms(m: Matcher_) -> Matcher { - Spanned { - node: m.clone(), - span: DUMMY_SP - } - } let lhs_nm = gensym_ident("lhs"); let rhs_nm = gensym_ident("rhs"); // The pattern that macro_rules matches. // The grammar for macro_rules! is: - // $( $lhs:mtcs => $rhs:tt );+ + // $( $lhs:tt => $rhs:tt );+ // ...quasiquoting this would be nice. + // These spans won't matter, anyways + let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain); + let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain); let argument_gram = vec!( - ms(MatchSeq(vec!( - ms(MatchNonterminal(lhs_nm, special_idents::matchers, 0u)), - ms(MatchTok(token::FatArrow)), - ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), - Some(token::Semi), ast::OneOrMore, 0u, 2u)), + TtSequence(DUMMY_SP, + Rc::new(ast::SequenceRepetition { + tts: vec![ + TtToken(DUMMY_SP, match_lhs_tok), + TtToken(DUMMY_SP, token::FatArrow), + TtToken(DUMMY_SP, match_rhs_tok)], + separator: Some(token::Semi), + op: ast::OneOrMore, + num_captures: 2 + })), //to phase into semicolon-termination instead of //semicolon-separation - ms(MatchSeq(vec!(ms(MatchTok(token::Semi))), None, - ast::ZeroOrMore, 2u, 2u))); + TtSequence(DUMMY_SP, + Rc::new(ast::SequenceRepetition { + tts: vec![TtToken(DUMMY_SP, token::Semi)], + separator: None, + op: ast::ZeroOrMore, + num_captures: 0 + }))); // Parse the macro_rules! invocation (`none` is for no interpolations): diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 249a985a6488a..5842afe11ce2c 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -9,10 +9,11 @@ // except according to those terms. use ast; -use ast::{TokenTree, TtDelimited, TtToken, TtSequence, TtNonterminal, Ident}; +use ast::{TokenTree, TtDelimited, TtToken, TtSequence, Ident}; use codemap::{Span, DUMMY_SP}; use diagnostic::SpanHandler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; +use parse::token::{Eof, DocComment, Interpolated, MatchNt, SubstNt}; use parse::token::{Token, NtIdent}; use parse::token; use parse::lexer::TokenAndSpan; @@ -24,7 +25,7 @@ use std::collections::HashMap; ///an unzipping of `TokenTree`s #[deriving(Clone)] struct TtFrame { - forest: Rc>, + forest: TokenTree, idx: uint, dotdotdoted: bool, sep: Option, @@ -42,6 +43,8 @@ pub struct TtReader<'a> { /* cached: */ pub cur_tok: Token, pub cur_span: Span, + /// Transform doc comments. Only useful in macro invocations + pub desugar_doc_comments: bool, } /// This can do Macro-By-Example transcription. On the other hand, if @@ -54,7 +57,11 @@ pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler, let mut r = TtReader { sp_diag: sp_diag, stack: vec!(TtFrame { - forest: Rc::new(src), + forest: TtSequence(DUMMY_SP, Rc::new(ast::SequenceRepetition { + tts: src, + // doesn't matter. This merely holds the root unzipping. + separator: None, op: ast::ZeroOrMore, num_captures: 0 + })), idx: 0, dotdotdoted: false, sep: None, @@ -65,6 +72,7 @@ pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler, }, repeat_idx: Vec::new(), repeat_len: Vec::new(), + desugar_doc_comments: false, /* dummy values, never read: */ cur_tok: token::Eof, cur_span: DUMMY_SP, @@ -85,17 +93,9 @@ fn lookup_cur_matched_by_matched(r: &TtReader, start: Rc) -> Rc Rc { +fn lookup_cur_matched(r: &TtReader, name: Ident) -> Option> { let matched_opt = r.interpolations.find_copy(&name); - match matched_opt { - Some(s) => lookup_cur_matched_by_matched(r, s), - None => { - r.sp_diag - .span_fatal(r.cur_span, - format!("unknown macro variable `{}`", - token::get_ident(name)).as_slice()); - } - } + matched_opt.map(|s| lookup_cur_matched_by_matched(r, s)) } #[deriving(Clone)] @@ -133,16 +133,20 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { size + lockstep_iter_size(tt, r) }) }, - TtSequence(_, ref tts, _, _) => { - tts.iter().fold(LisUnconstrained, |size, tt| { + TtSequence(_, ref seq) => { + seq.tts.iter().fold(LisUnconstrained, |size, tt| { size + lockstep_iter_size(tt, r) }) }, + TtToken(_, SubstNt(name, _)) | TtToken(_, MatchNt(name, _, _, _)) => + match lookup_cur_matched(r, name) { + Some(matched) => match *matched { + MatchedNonterminal(_) => LisUnconstrained, + MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name), + }, + _ => LisUnconstrained + }, TtToken(..) => LisUnconstrained, - TtNonterminal(_, name) => match *lookup_cur_matched(r, name) { - MatchedNonterminal(_) => LisUnconstrained, - MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name) - }, } } @@ -202,46 +206,27 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { let t = { let frame = r.stack.last().unwrap(); // FIXME(pcwalton): Bad copy. - (*frame.forest)[frame.idx].clone() + frame.forest.get_tt(frame.idx) }; match t { - TtDelimited(_, ref delimed) => { - let mut tts = Vec::with_capacity(1 + delimed.tts.len() + 1); - tts.push(delimed.open_tt()); - tts.extend(delimed.tts.iter().map(|tt| tt.clone())); - tts.push(delimed.close_tt()); - - r.stack.push(TtFrame { - forest: Rc::new(tts), - idx: 0, - dotdotdoted: false, - sep: None - }); - // if this could be 0-length, we'd need to potentially recur here - } - TtToken(sp, tok) => { - r.cur_span = sp; - r.cur_tok = tok; - r.stack.last_mut().unwrap().idx += 1; - return ret_val; - } - TtSequence(sp, tts, sep, kleene_op) => { + TtSequence(sp, seq) => { // FIXME(pcwalton): Bad copy. - match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), kleene_op), r) { + match lockstep_iter_size(&TtSequence(sp, seq.clone()), + r) { LisUnconstrained => { r.sp_diag.span_fatal( sp.clone(), /* blame macro writer */ "attempted to repeat an expression \ containing no syntax \ variables matched as repeating at this depth"); - } - LisContradiction(ref msg) => { - // FIXME #2887 blame macro invoker instead - r.sp_diag.span_fatal(sp.clone(), msg.as_slice()); - } + } + LisContradiction(ref msg) => { + // FIXME #2887 blame macro invoker instead + r.sp_diag.span_fatal(sp.clone(), msg.as_slice()); + } LisConstraint(len, _) => { if len == 0 { - if kleene_op == ast::OneOrMore { + if seq.op == ast::OneOrMore { // FIXME #2887 blame invoker r.sp_diag.span_fatal(sp.clone(), "this must repeat at least once"); @@ -253,40 +238,78 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { r.repeat_len.push(len); r.repeat_idx.push(0); r.stack.push(TtFrame { - forest: tts, idx: 0, dotdotdoted: true, - sep: sep.clone() + sep: seq.separator.clone(), + forest: TtSequence(sp, seq), }); } } } // FIXME #2887: think about span stuff here - TtNonterminal(sp, ident) => { - r.stack.last_mut().unwrap().idx += 1; - match *lookup_cur_matched(r, ident) { - /* sidestep the interpolation tricks for ident because - (a) idents can be in lots of places, so it'd be a pain - (b) we actually can, since it's a token. */ - MatchedNonterminal(NtIdent(box sn, b)) => { - r.cur_span = sp; - r.cur_tok = token::Ident(sn,b); - return ret_val; - } - MatchedNonterminal(ref other_whole_nt) => { - // FIXME(pcwalton): Bad copy. - r.cur_span = sp; - r.cur_tok = token::Interpolated((*other_whole_nt).clone()); - return ret_val; + TtToken(sp, SubstNt(ident, namep)) => { + match lookup_cur_matched(r, ident) { + None => { + r.stack.push(TtFrame { + forest: TtToken(sp, SubstNt(ident, namep)), + idx: 0, + dotdotdoted: false, + sep: None + }); + // this can't be 0 length, just like TtDelimited } - MatchedSeq(..) => { - r.sp_diag.span_fatal( - r.cur_span, /* blame the macro writer */ - format!("variable '{}' is still repeating at this depth", - token::get_ident(ident)).as_slice()); + Some(cur_matched) => { + r.stack.last_mut().unwrap().idx += 1; + match *cur_matched { + // sidestep the interpolation tricks for ident because + // (a) idents can be in lots of places, so it'd be a pain + // (b) we actually can, since it's a token. + MatchedNonterminal(NtIdent(box sn, b)) => { + r.cur_span = sp; + r.cur_tok = token::Ident(sn, b); + return ret_val; + } + MatchedNonterminal(ref other_whole_nt) => { + // FIXME(pcwalton): Bad copy. + r.cur_span = sp; + r.cur_tok = token::Interpolated((*other_whole_nt).clone()); + return ret_val; + } + MatchedSeq(..) => { + r.sp_diag.span_fatal( + r.cur_span, /* blame the macro writer */ + format!("variable '{}' is still repeating at this depth", + token::get_ident(ident)).as_slice()); + } + } } } } + // TtDelimited or any token that can be unzipped + seq @ TtDelimited(..) | seq @ TtToken(_, MatchNt(..)) => { + // do not advance the idx yet + r.stack.push(TtFrame { + forest: seq, + idx: 0, + dotdotdoted: false, + sep: None + }); + // if this could be 0-length, we'd need to potentially recur here + } + TtToken(sp, DocComment(name)) if r.desugar_doc_comments => { + r.stack.push(TtFrame { + forest: TtToken(sp, DocComment(name)), + idx: 0, + dotdotdoted: false, + sep: None + }); + } + TtToken(sp, tok) => { + r.cur_span = sp; + r.cur_tok = tok; + r.stack.last_mut().unwrap().idx += 1; + return ret_val; + } } } } diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index cd4a3d10c4882..2074a6bfab982 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -619,13 +619,13 @@ pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { } )) }, - TtSequence(span, ref pattern, ref sep, is_optional) => + TtSequence(span, ref seq) => TtSequence(span, - Rc::new(fld.fold_tts(pattern.as_slice())), - sep.clone().map(|tok| fld.fold_token(tok)), - is_optional), - TtNonterminal(sp,ref ident) => - TtNonterminal(sp,fld.fold_ident(*ident)) + Rc::new(SequenceRepetition { + tts: fld.fold_tts(seq.tts.as_slice()), + separator: seq.separator.clone().map(|tok| fld.fold_token(tok)), + ..**seq + })), } } @@ -641,6 +641,12 @@ pub fn noop_fold_token(t: token::Token, fld: &mut T) -> token::Token } token::Lifetime(id) => token::Lifetime(fld.fold_ident(id)), token::Interpolated(nt) => token::Interpolated(fld.fold_interpolated(nt)), + token::SubstNt(ident, namep) => { + token::SubstNt(fld.fold_ident(ident), namep) + } + token::MatchNt(name, kind, namep, kindp) => { + token::MatchNt(fld.fold_ident(name), fld.fold_ident(kind), namep, kindp) + } _ => t } } @@ -689,8 +695,6 @@ pub fn noop_fold_interpolated(nt: token::Nonterminal, fld: &mut T) token::NtMeta(meta_item) => token::NtMeta(fld.fold_meta_item(meta_item)), token::NtPath(box path) => token::NtPath(box fld.fold_path(path)), token::NtTT(tt) => token::NtTT(P(fld.fold_tt(&*tt))), - // it looks to me like we can leave out the matchers: token::NtMatchers(matchers) - _ => nt } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 18dd7074d28b3..9ba11d63da54f 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -37,8 +37,8 @@ use ast::{ItemMac, ItemMod, ItemStruct, ItemTrait, ItemTy}; use ast::{LifetimeDef, Lit, Lit_}; use ast::{LitBool, LitChar, LitByte, LitBinary}; use ast::{LitNil, LitStr, LitInt, Local, LocalLet}; -use ast::{MutImmutable, MutMutable, Mac_, MacInvocTT, Matcher, MatchNonterminal, MatchNormal}; -use ast::{MatchSeq, MatchTok, Method, MutTy, BiMul, Mutability}; +use ast::{MutImmutable, MutMutable, Mac_, MacInvocTT, MatchNormal}; +use ast::{Method, MutTy, BiMul, Mutability}; use ast::{MethodImplItem, NamedField, UnNeg, NoReturn, UnNot}; use ast::{Pat, PatEnum, PatIdent, PatLit, PatRange, PatRegion, PatStruct}; use ast::{PatTup, PatBox, PatWild, PatWildMulti, PatWildSingle}; @@ -48,8 +48,9 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField}; use ast::{StructVariantKind, BiSub}; use ast::StrStyle; use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue}; -use ast::{Delimited, TokenTree, TraitItem, TraitRef, TtDelimited, TtSequence, TtToken}; -use ast::{TtNonterminal, TupleVariantKind, Ty, Ty_, TyBot}; +use ast::{Delimited, SequenceRepetition, TokenTree, TraitItem, TraitRef}; +use ast::{TtDelimited, TtSequence, TtToken}; +use ast::{TupleVariantKind, Ty, Ty_, TyBot}; use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn}; use ast::{TyTypeof, TyInfer, TypeMethod}; use ast::{TyNil, TyParam, TyParamBound, TyParen, TyPath, TyPtr, TyQPath}; @@ -64,6 +65,7 @@ use ast_util::{as_prec, ident_to_path, operator_prec}; use ast_util; use codemap::{Span, BytePos, Spanned, spanned, mk_sp}; use codemap; +use ext::tt::macro_parser; use parse; use parse::attr::ParserAttr; use parse::classify; @@ -72,7 +74,7 @@ use parse::common::{seq_sep_trailing_allowed}; use parse::lexer::Reader; use parse::lexer::TokenAndSpan; use parse::obsolete::*; -use parse::token::InternedString; +use parse::token::{MatchNt, SubstNt, InternedString}; use parse::token::{keywords, special_idents}; use parse::token; use parse::{new_sub_parser_from_file, ParseSess}; @@ -2579,7 +2581,7 @@ impl<'a> Parser<'a> { pub fn parse_token_tree(&mut self) -> TokenTree { // FIXME #6994: currently, this is too eager. It // parses token trees but also identifies TtSequence's - // and TtNonterminal's; it's too early to know yet + // and token::SubstNt's; it's too early to know yet // whether something will be a nonterminal or a seq // yet. maybe_whole!(deref self, NtTT); @@ -2620,9 +2622,27 @@ impl<'a> Parser<'a> { let seq = match seq { Spanned { node, .. } => node, }; - TtSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), sep, repeat) + let name_num = macro_parser::count_names(seq.as_slice()); + TtSequence(mk_sp(sp.lo, p.span.hi), + Rc::new(SequenceRepetition { + tts: seq, + separator: sep, + op: repeat, + num_captures: name_num + })) } else { - TtNonterminal(sp, p.parse_ident()) + // A nonterminal that matches or not + let namep = match p.token { token::Ident(_, p) => p, _ => token::Plain }; + let name = p.parse_ident(); + if p.token == token::Colon && p.look_ahead(1, |t| t.is_ident()) { + p.bump(); + let kindp = match p.token { token::Ident(_, p) => p, _ => token::Plain }; + let nt_kind = p.parse_ident(); + let m = TtToken(sp, MatchNt(name, nt_kind, namep, kindp)); + m + } else { + TtToken(sp, SubstNt(name, namep)) + } } } _ => { @@ -2686,66 +2706,6 @@ impl<'a> Parser<'a> { tts } - pub fn parse_matchers(&mut self) -> Vec { - // unification of Matcher's and TokenTree's would vastly improve - // the interpolation of Matcher's - maybe_whole!(self, NtMatchers); - let mut name_idx = 0u; - let delim = self.expect_open_delim(); - self.parse_matcher_subseq_upto(&mut name_idx, &token::CloseDelim(delim)) - } - - /// This goofy function is necessary to correctly match parens in Matcher's. - /// Otherwise, `$( ( )` would be a valid Matcher, and `$( () )` would be - /// invalid. It's similar to common::parse_seq. - pub fn parse_matcher_subseq_upto(&mut self, - name_idx: &mut uint, - ket: &token::Token) - -> Vec { - let mut ret_val = Vec::new(); - let mut lparens = 0u; - - while self.token != *ket || lparens > 0u { - if self.token == token::OpenDelim(token::Paren) { lparens += 1u; } - if self.token == token::CloseDelim(token::Paren) { lparens -= 1u; } - ret_val.push(self.parse_matcher(name_idx)); - } - - self.bump(); - - return ret_val; - } - - pub fn parse_matcher(&mut self, name_idx: &mut uint) -> Matcher { - let lo = self.span.lo; - - let m = if self.token == token::Dollar { - self.bump(); - if self.token == token::OpenDelim(token::Paren) { - let name_idx_lo = *name_idx; - self.bump(); - let ms = self.parse_matcher_subseq_upto(name_idx, - &token::CloseDelim(token::Paren)); - if ms.len() == 0u { - self.fatal("repetition body must be nonempty"); - } - let (sep, kleene_op) = self.parse_sep_and_kleene_op(); - MatchSeq(ms, sep, kleene_op, name_idx_lo, *name_idx) - } else { - let bound_to = self.parse_ident(); - self.expect(&token::Colon); - let nt_name = self.parse_ident(); - let m = MatchNonterminal(bound_to, nt_name, *name_idx); - *name_idx += 1; - m - } - } else { - MatchTok(self.bump_and_get()) - }; - - return spanned(lo, self.span.hi, m); - } - /// Parse a prefix-operator expr pub fn parse_prefix_expr(&mut self) -> P { let lo = self.span.lo; diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 29a2f38959eed..b0cca5e14de15 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -108,7 +108,15 @@ pub enum Token { /* For interpolation */ Interpolated(Nonterminal), + // Can be expanded into several tokens. + /// Doc comment DocComment(ast::Name), + // In left-hand-sides of MBE macros: + /// Parse a nonterminal (name to bind, name of NT, styles of their idents) + MatchNt(ast::Ident, ast::Ident, IdentStyle, IdentStyle), + // In right-hand-sides of MBE macros: + /// A syntactic variable that will be filled in by macro expansion. + SubstNt(ast::Ident, IdentStyle), // Junk. These carry no data because we don't really care about the data // they *would* carry, and don't really want to allocate a new ident for @@ -329,7 +337,6 @@ pub enum Nonterminal { NtMeta(P), NtPath(Box), NtTT(P), // needs P'ed to break a circularity - NtMatchers(Vec) } impl fmt::Show for Nonterminal { @@ -345,7 +352,6 @@ impl fmt::Show for Nonterminal { NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), NtTT(..) => f.pad("NtTT(..)"), - NtMatchers(..) => f.pad("NtMatchers(..)"), } } } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 2448eacbb3998..63e2c5499e8f1 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -254,6 +254,8 @@ pub fn token_to_string(tok: &Token) -> String { /* Other */ token::DocComment(s) => s.as_str().into_string(), + token::SubstNt(s, _) => format!("${}", s), + token::MatchNt(s, t, _, _) => format!("${}:{}", s, t), token::Eof => "".into_string(), token::Whitespace => " ".into_string(), token::Comment => "/* */".into_string(), @@ -270,7 +272,6 @@ pub fn token_to_string(tok: &Token) -> String { token::NtPat(..) => "an interpolated pattern".into_string(), token::NtIdent(..) => "an interpolated identifier".into_string(), token::NtTT(..) => "an interpolated tt".into_string(), - token::NtMatchers(..) => "an interpolated matcher sequence".into_string(), } } } @@ -1105,13 +1106,6 @@ impl<'a> State<'a> { /// expression arguments as expressions). It can be done! I think. pub fn print_tt(&mut self, tt: &ast::TokenTree) -> IoResult<()> { match *tt { - ast::TtDelimited(_, ref delimed) => { - try!(word(&mut self.s, token_to_string(&delimed.open_token()).as_slice())); - try!(space(&mut self.s)); - try!(self.print_tts(delimed.tts.as_slice())); - try!(space(&mut self.s)); - word(&mut self.s, token_to_string(&delimed.close_token()).as_slice()) - }, ast::TtToken(_, ref tk) => { try!(word(&mut self.s, token_to_string(tk).as_slice())); match *tk { @@ -1121,27 +1115,30 @@ impl<'a> State<'a> { _ => Ok(()) } } - ast::TtSequence(_, ref tts, ref separator, kleene_op) => { + ast::TtDelimited(_, ref delimed) => { + try!(word(&mut self.s, token_to_string(&delimed.open_token()).as_slice())); + try!(space(&mut self.s)); + try!(self.print_tts(delimed.tts.as_slice())); + try!(space(&mut self.s)); + word(&mut self.s, token_to_string(&delimed.close_token()).as_slice()) + }, + ast::TtSequence(_, ref seq) => { try!(word(&mut self.s, "$(")); - for tt_elt in (*tts).iter() { + for tt_elt in seq.tts.iter() { try!(self.print_tt(tt_elt)); } try!(word(&mut self.s, ")")); - match *separator { + match seq.separator { Some(ref tk) => { try!(word(&mut self.s, token_to_string(tk).as_slice())); } None => {}, } - match kleene_op { + match seq.op { ast::ZeroOrMore => word(&mut self.s, "*"), ast::OneOrMore => word(&mut self.s, "+"), } } - ast::TtNonterminal(_, name) => { - try!(word(&mut self.s, "$")); - self.print_ident(name) - } } } diff --git a/src/test/compile-fail/issue-6596.rs b/src/test/compile-fail/issue-6596.rs index 54bcf415d2ead..267b30677a18f 100644 --- a/src/test/compile-fail/issue-6596.rs +++ b/src/test/compile-fail/issue-6596.rs @@ -10,7 +10,7 @@ #![feature(macro_rules)] -// error-pattern: unknown macro variable `nonexistent` +// error-pattern: unexpected token macro_rules! e( ($inp:ident) => ( diff --git a/src/test/compile-fail/macro-match-nonterminal.rs b/src/test/compile-fail/macro-match-nonterminal.rs new file mode 100644 index 0000000000000..d6d32d94a2956 --- /dev/null +++ b/src/test/compile-fail/macro-match-nonterminal.rs @@ -0,0 +1,17 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(macro_rules)] + +macro_rules! test ( ($a, $b) => (()); ) //~ ERROR Cannot transcribe + +fn main() { + test!() +} diff --git a/src/test/compile-fail/issue-5060-fail.rs b/src/test/run-pass/macro-of-higher-order.rs similarity index 51% rename from src/test/compile-fail/issue-5060-fail.rs rename to src/test/run-pass/macro-of-higher-order.rs index fedb064d2e5f5..561933d759927 100644 --- a/src/test/compile-fail/issue-5060-fail.rs +++ b/src/test/run-pass/macro-of-higher-order.rs @@ -1,4 +1,4 @@ -// Copyright 2013 The Rust Project Developers. See the COPYRIGHT +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // @@ -10,20 +10,14 @@ #![feature(macro_rules)] -macro_rules! print_hd_tl ( - ($field_hd:ident, $($field_tl:ident),+) => ({ - print!("{}", stringify!($field)); //~ ERROR unknown macro variable - print!("::["); - $( - print!("{}", stringify!($field_tl)); - print!(", "); - )+ - // FIXME: #9970 - print!("{}", "]\n"); - }) +macro_rules! higher_order ( + (subst $lhs:tt => $rhs:tt) => ({ + macro_rules! anon ( $lhs => $rhs ) + anon!(1u, 2u, "foo") + }); ) fn main() { - print_hd_tl!(x, y, z, w) + let val = higher_order!(subst ($x:expr, $y:expr, $foo:expr) => (($x + $y, $foo))); + assert_eq!(val, (3, "foo")); } -