Skip to content

Commit

Permalink
auto merge of rust-lang#17830 : pczarn/rust/interp_tt, r=pnkfelix
Browse files Browse the repository at this point in the history 
Closes rust-lang#14197

Removes the `matchers` nonterminal.

If you're using `$foo:matchers` in a macro, write `$foo:tt` instead.

[breaking-change]
  • Loading branch information
@bors
bors committed Nov 7, 2014
2 parents 223ca76 + 00676c8 commit 0b48001
Show file tree
Hide file tree
Showing 13 changed files with 539 additions and 435 deletions.
3 changes: 2 additions & 1 deletion src/librustdoc/html/highlight.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,8 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,

token::Lifetime(..) => "lifetime",
token::DocComment(..) => "doccomment",
token::Underscore | token::Eof | token::Interpolated(..) => "",
token::Underscore | token::Eof | token::Interpolated(..) |
token::MatchNt(..) | token::SubstNt(..) => "",
};

// as mentioned above, use the original source code instead of
Expand Down
163 changes: 83 additions & 80 deletions src/libsyntax/ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

// The Rust abstract syntax tree.

use codemap::{Span, Spanned, DUMMY_SP, ExpnId};
use codemap::{Span, Spanned, DUMMY_SP, ExpnId, respan};
use abi::Abi;
use ast_util;
use owned_slice::OwnedSlice;
Expand Down Expand Up @@ -713,6 +713,19 @@ impl Delimited {
}
}

/// A sequence of token treesee
#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
pub struct SequenceRepetition {
/// The sequence of token trees
pub tts: Vec<TokenTree>,
/// The optional separator
pub separator: Option<token::Token>,
/// Whether the sequence can be repeated zero (*), or one or more times (+)
pub op: KleeneOp,
/// The number of `MatchNt`s that appear in the sequence (and subsequences)
pub num_captures: uint,
}

/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
/// for token sequences.
#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
Expand All @@ -727,14 +740,12 @@ pub enum KleeneOp {
/// be passed to syntax extensions using a uniform type.
///
/// If the syntax extension is an MBE macro, it will attempt to match its
/// LHS "matchers" against the provided token tree, and if it finds a
/// LHS token tree against the provided token tree, and if it finds a
/// match, will transcribe the RHS token tree, splicing in any captured
/// `macro_parser::matched_nonterminals` into the `TtNonterminal`s it finds.
/// macro_parser::matched_nonterminals into the `SubstNt`s it finds.
///
/// The RHS of an MBE macro is the only place a `TtNonterminal` or `TtSequence`
/// makes any real sense. You could write them elsewhere but nothing
/// else knows what to do with them, so you'll probably get a syntax
/// error.
/// The RHS of an MBE macro is the only place `SubstNt`s are substituted.
/// Nothing special happens to misnamed or misplaced `SubstNt`s.
#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
#[doc="For macro invocations; parsing is delegated to the macro"]
pub enum TokenTree {
Expand All @@ -743,90 +754,82 @@ pub enum TokenTree {
/// A delimited sequence of token trees
TtDelimited(Span, Rc<Delimited>),

// These only make sense for right-hand-sides of MBE macros:
// This only makes sense in MBE macros.

/// A Kleene-style repetition sequence with an optional separator.
// FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST.
TtSequence(Span, Rc<Vec<TokenTree>>, Option<token::Token>, KleeneOp),
/// A syntactic variable that will be filled in by macro expansion.
TtNonterminal(Span, Ident)
/// A kleene-style repetition sequence with a span
// FIXME(eddyb) #12938 Use DST.
TtSequence(Span, Rc<SequenceRepetition>),
}

impl TokenTree {
pub fn len(&self) -> uint {
match *self {
TtToken(_, token::DocComment(_)) => 2,
TtToken(_, token::SubstNt(..)) => 2,
TtToken(_, token::MatchNt(..)) => 3,
TtDelimited(_, ref delimed) => {
delimed.tts.len() + 2
}
TtSequence(_, ref seq) => {
seq.tts.len()
}
TtToken(..) => 0
}
}

pub fn get_tt(&self, index: uint) -> TokenTree {
match (self, index) {
(&TtToken(sp, token::DocComment(_)), 0) => {
TtToken(sp, token::Pound)
}
(&TtToken(sp, token::DocComment(name)), 1) => {
let doc = MetaNameValue(token::intern_and_get_ident("doc"),
respan(sp, LitStr(token::get_name(name), CookedStr)));
let doc = token::NtMeta(P(respan(sp, doc)));
TtDelimited(sp, Rc::new(Delimited {
delim: token::Bracket,
open_span: sp,
tts: vec![TtToken(sp, token::Interpolated(doc))],
close_span: sp,
}))
}
(&TtDelimited(_, ref delimed), _) => {
if index == 0 {
return delimed.open_tt();
}
if index == delimed.tts.len() + 1 {
return delimed.close_tt();
}
delimed.tts[index - 1].clone()
}
(&TtToken(sp, token::SubstNt(name, name_st)), _) => {
let v = [TtToken(sp, token::Dollar),
TtToken(sp, token::Ident(name, name_st))];
v[index]
}
(&TtToken(sp, token::MatchNt(name, kind, name_st, kind_st)), _) => {
let v = [TtToken(sp, token::SubstNt(name, name_st)),
TtToken(sp, token::Colon),
TtToken(sp, token::Ident(kind, kind_st))];
v[index]
}
(&TtSequence(_, ref seq), _) => {
seq.tts[index].clone()
}
_ => panic!("Cannot expand a token tree")
}
}

/// Returns the `Span` corresponding to this token tree.
pub fn get_span(&self) -> Span {
match *self {
TtToken(span, _) => span,
TtDelimited(span, _) => span,
TtSequence(span, _, _, _) => span,
TtNonterminal(span, _) => span,
TtToken(span, _) => span,
TtDelimited(span, _) => span,
TtSequence(span, _) => span,
}
}
}

// Matchers are nodes defined-by and recognized-by the main rust parser and
// language, but they're only ever found inside syntax-extension invocations;
// indeed, the only thing that ever _activates_ the rules in the rust parser
// for parsing a matcher is a matcher looking for the 'matchers' nonterminal
// itself. Matchers represent a small sub-language for pattern-matching
// token-trees, and are thus primarily used by the macro-defining extension
// itself.
//
// MatchTok
// --------
//
// A matcher that matches a single token, denoted by the token itself. So
// long as there's no $ involved.
//
//
// MatchSeq
// --------
//
// A matcher that matches a sequence of sub-matchers, denoted various
// possible ways:
//
// $(M)* zero or more Ms
// $(M)+ one or more Ms
// $(M),+ one or more comma-separated Ms
// $(A B C);* zero or more semi-separated 'A B C' seqs
//
//
// MatchNonterminal
// -----------------
//
// A matcher that matches one of a few interesting named rust
// nonterminals, such as types, expressions, items, or raw token-trees. A
// black-box matcher on expr, for example, binds an expr to a given ident,
// and that ident can re-occur as an interpolation in the RHS of a
// macro-by-example rule. For example:
//
// $foo:expr => 1 + $foo // interpolate an expr
// $foo:tt => $foo // interpolate a token-tree
// $foo:tt => bar! $foo // only other valid interpolation
// // is in arg position for another
// // macro
//
// As a final, horrifying aside, note that macro-by-example's input is
// also matched by one of these matchers. Holy self-referential! It is matched
// by a MatchSeq, specifically this one:
//
// $( $lhs:matchers => $rhs:tt );+
//
// If you understand that, you have closed the loop and understand the whole
// macro system. Congratulations.
pub type Matcher = Spanned<Matcher_>;

#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
pub enum Matcher_ {
/// Match one token
MatchTok(token::Token),
/// Match repetitions of a sequence: body, separator, Kleene operator,
/// lo, hi position-in-match-array used:
MatchSeq(Vec<Matcher>, Option<token::Token>, KleeneOp, uint, uint),
/// Parse a Rust NT: name to bind, name of NT, position in match array:
MatchNonterminal(Ident, Ident, uint)
}

pub type Mac = Spanned<Mac_>;

/// Represents a macro invocation. The Path indicates which macro
Expand Down
52 changes: 33 additions & 19 deletions src/libsyntax/ext/quote.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,20 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
vec!(mk_name(cx, sp, ident.ident())));
}

token::MatchNt(name, kind, name_style, kind_style) => {
return cx.expr_call(sp,
mk_token_path(cx, sp, "MatchNt"),
vec![mk_ident(cx, sp, name),
mk_ident(cx, sp, kind),
match name_style {
ModName => mk_token_path(cx, sp, "ModName"),
Plain => mk_token_path(cx, sp, "Plain"),
},
match kind_style {
ModName => mk_token_path(cx, sp, "ModName"),
Plain => mk_token_path(cx, sp, "Plain"),
}]);
}
token::Interpolated(_) => panic!("quote! with interpolated token"),

_ => ()
Expand Down Expand Up @@ -654,6 +668,25 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {

fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
match *tt {
ast::TtToken(sp, SubstNt(ident, _)) => {
// tt.extend($ident.to_tokens(ext_cx).into_iter())

let e_to_toks =
cx.expr_method_call(sp,
cx.expr_ident(sp, ident),
id_ext("to_tokens"),
vec!(cx.expr_ident(sp, id_ext("ext_cx"))));
let e_to_toks =
cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]);

let e_push =
cx.expr_method_call(sp,
cx.expr_ident(sp, id_ext("tt")),
id_ext("extend"),
vec!(e_to_toks));

vec!(cx.stmt_expr(e_push))
}
ast::TtToken(sp, ref tok) => {
let e_sp = cx.expr_ident(sp, id_ext("_sp"));
let e_tok = cx.expr_call(sp,
Expand All @@ -673,25 +706,6 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
.collect()
},
ast::TtSequence(..) => panic!("TtSequence in quote!"),
ast::TtNonterminal(sp, ident) => {
// tt.extend($ident.to_tokens(ext_cx).into_iter())

let e_to_toks =
cx.expr_method_call(sp,
cx.expr_ident(sp, ident),
id_ext("to_tokens"),
vec!(cx.expr_ident(sp, id_ext("ext_cx"))));
let e_to_toks =
cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]);

let e_push =
cx.expr_method_call(sp,
cx.expr_ident(sp, id_ext("tt")),
id_ext("extend"),
vec!(e_to_toks));

vec!(cx.stmt_expr(e_push))
},
}
}

Expand Down
Loading

0 comments on commit 0b48001

Please sign in to comment.