Skip to content

Commit

Permalink
Add the %extra_context directive to lemon, as an alternative to %extr…
Browse files Browse the repository at this point in the history 
…a_argument.

Use this to improve the performance of the parser.
  • Loading branch information
D. Richard Hipp committed Apr 21, 2018
1 parent b3b0ea8 commit 2ead4b6
Show file tree
Hide file tree
Showing 6 changed files with 113 additions and 44 deletions.
27 changes: 27 additions & 0 deletions doc/lemon.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ <h3>Command Line Options</h3>
<li><b>-c</b>
Do not compress the generated action tables. The parser will be a
little larger and slower, but it will detect syntax errors sooner.
<li><b>-d</b><i>directory</i>
Write all output files into <i>directory</i>. Normally, output files
are written into the directory that contains the input grammar file.
<li><b>-D<i>name</i></b>
Define C preprocessor macro <i>name</i>. This macro is usable by
"<tt><a href='#pifdef'>%ifdef</a></tt>" and
Expand Down Expand Up @@ -679,6 +682,30 @@ <h4>The <tt>%extra_argument</tt> directive</h4>
a variable named "pAbc" that is the value of the 4th parameter
in the most recent call to Parse().</p>

<p>The <tt>%extra_context</tt> directive works the same except that it
is passed in on the ParseAlloc() or ParseInit() routines instead of
on Parse().

<a name='extractx'></a>
<h4>The <tt>%extra_context</tt> directive</h4>

The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
to the parameter list of the ParseAlloc() and ParseInif() functions. Lemon
doesn't do anything itself with these extra argument, but it does
store the value make it available to C-code action routines, destructors,
and so forth. For example, if the grammar file contains:</p>

<p><pre>
%extra_context { MyStruct *pAbc }
</pre></p>

<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
of type "MyStruct*" and all action routines will have access to
a variable named "pAbc" that is the value of that 2th parameter.</p>

<p>The <tt>%extra_argument</tt> directive works the same except that it
is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().

<a name='pfallback'></a>
<h4>The <tt>%fallback</tt> directive</h4>

Expand Down
5 changes: 3 additions & 2 deletions src/parse.y
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@
%token_type {Token}
%default_type {Token}

// The generated parser function takes a 4th argument as follows:
%extra_argument {Parse *pParse}
// An extra argument to the constructor for the parser, which is available
// to all actions.
%extra_context {Parse *pParse}

// This code runs whenever there is a syntax error
//
Expand Down
4 changes: 2 additions & 2 deletions src/sqliteInt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4213,10 +4213,10 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
** The interface to the LEMON-generated parser
*/
#ifndef SQLITE_AMALGAMATION
void *sqlite3ParserAlloc(void*(*)(u64));
void *sqlite3ParserAlloc(void*(*)(u64), Parse*);
void sqlite3ParserFree(void*, void(*)(void*));
#endif
void sqlite3Parser(void*, int, Token, Parse*);
void sqlite3Parser(void*, int, Token);
#ifdef YYTRACKMAXSTACKDEPTH
int sqlite3ParserStackPeak(void*);
#endif
Expand Down
6 changes: 3 additions & 3 deletions src/tokenize.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,9 +496,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
/* sqlite3ParserTrace(stdout, "parser: "); */
#ifdef sqlite3Parser_ENGINEALWAYSONSTACK
pEngine = &sEngine;
sqlite3ParserInit(pEngine);
sqlite3ParserInit(pEngine, pParse);
#else
pEngine = sqlite3ParserAlloc(sqlite3Malloc);
pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse);
if( pEngine==0 ){
sqlite3OomFault(db);
return SQLITE_NOMEM_BKPT;
Expand Down Expand Up @@ -542,7 +542,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
}else{
pParse->sLastToken.z = zSql;
pParse->sLastToken.n = n;
sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
lastTokenParsed = tokenType;
zSql += n;
if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
Expand Down
32 changes: 28 additions & 4 deletions tool/lemon.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,7 @@ struct lemon {
struct symbol *wildcard; /* Token that matches anything */
char *name; /* Name of the generated parser */
char *arg; /* Declaration of the 3th argument to parser */
char *ctx; /* Declaration of 2nd argument to constructor */
char *tokentype; /* Type of terminal symbols in the parser stack */
char *vartype; /* The default type of non-terminal symbols */
char *start; /* Name of the start symbol for the grammar */
Expand Down Expand Up @@ -2493,6 +2494,9 @@ to follow the previous rule.");
}else if( strcmp(x,"extra_argument")==0 ){
psp->declargslot = &(psp->gp->arg);
psp->insertLineMacro = 0;
}else if( strcmp(x,"extra_context")==0 ){
psp->declargslot = &(psp->gp->ctx);
psp->insertLineMacro = 0;
}else if( strcmp(x,"token_type")==0 ){
psp->declargslot = &(psp->gp->tokentype);
psp->insertLineMacro = 0;
Expand Down Expand Up @@ -4198,16 +4202,36 @@ void ReportTable(
while( i>=1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--;
fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++;
fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg); lineno++;
fprintf(out,"#define %sARG_FETCH %s = yypParser->%s\n",
fprintf(out,"#define %sARG_PARAM ,%s\n",name,&lemp->arg[i]); lineno++;
fprintf(out,"#define %sARG_FETCH %s=yypParser->%s;\n",
name,lemp->arg,&lemp->arg[i]); lineno++;
fprintf(out,"#define %sARG_STORE yypParser->%s = %s\n",
fprintf(out,"#define %sARG_STORE yypParser->%s=%s;\n",
name,&lemp->arg[i],&lemp->arg[i]); lineno++;
}else{
fprintf(out,"#define %sARG_SDECL\n",name); lineno++;
fprintf(out,"#define %sARG_PDECL\n",name); lineno++;
fprintf(out,"#define %sARG_SDECL\n",name); lineno++;
fprintf(out,"#define %sARG_PDECL\n",name); lineno++;
fprintf(out,"#define %sARG_PARAM\n",name); lineno++;
fprintf(out,"#define %sARG_FETCH\n",name); lineno++;
fprintf(out,"#define %sARG_STORE\n",name); lineno++;
}
if( lemp->ctx && lemp->ctx[0] ){
i = lemonStrlen(lemp->ctx);
while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--;
while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--;
fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++;
fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++;
fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++;
fprintf(out,"#define %sCTX_FETCH %s=yypParser->%s;\n",
name,lemp->ctx,&lemp->ctx[i]); lineno++;
fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n",
name,&lemp->ctx[i],&lemp->ctx[i]); lineno++;
}else{
fprintf(out,"#define %sCTX_SDECL\n",name); lineno++;
fprintf(out,"#define %sCTX_PDECL\n",name); lineno++;
fprintf(out,"#define %sCTX_PARAM\n",name); lineno++;
fprintf(out,"#define %sCTX_FETCH\n",name); lineno++;
fprintf(out,"#define %sCTX_STORE\n",name); lineno++;
}
if( mhflag ){
fprintf(out,"#endif\n"); lineno++;
}
Expand Down
83 changes: 50 additions & 33 deletions tool/lempar.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,10 @@
** zero the stack is dynamically sized using realloc()
** ParseARG_SDECL A static variable declaration for the %extra_argument
** ParseARG_PDECL A parameter declaration for the %extra_argument
** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter
** ParseARG_STORE Code to store %extra_argument into yypParser
** ParseARG_FETCH Code to extract %extra_argument from yypParser
** ParseCTX_* As ParseARG_ except for %extra_context
** YYERRORSYMBOL is the code number of the error symbol. If not
** defined, then do no error processing.
** YYNSTATE the combined number of states.
Expand Down Expand Up @@ -211,6 +213,7 @@ struct yyParser {
int yyerrcnt; /* Shifts left before out of the error */
#endif
ParseARG_SDECL /* A place to hold %extra_argument */
ParseCTX_SDECL /* A place to hold %extra_context */
#if YYSTACKDEPTH<=0
int yystksz; /* Current side of the stack */
yyStackEntry *yystack; /* The parser's stack */
Expand Down Expand Up @@ -315,28 +318,29 @@ static int yyGrowStack(yyParser *p){

/* Initialize a new parser that has already been allocated.
*/
void ParseInit(void *yypParser){
yyParser *pParser = (yyParser*)yypParser;
void ParseInit(void *yypRawParser ParseCTX_PDECL){
yyParser *yypParser = (yyParser*)yypRawParser;
ParseCTX_STORE
#ifdef YYTRACKMAXSTACKDEPTH
pParser->yyhwm = 0;
yypParser->yyhwm = 0;
#endif
#if YYSTACKDEPTH<=0
pParser->yytos = NULL;
pParser->yystack = NULL;
pParser->yystksz = 0;
if( yyGrowStack(pParser) ){
pParser->yystack = &pParser->yystk0;
pParser->yystksz = 1;
yypParser->yytos = NULL;
yypParser->yystack = NULL;
yypParser->yystksz = 0;
if( yyGrowStack(yypParser) ){
yypParser->yystack = &yypParser->yystk0;
yypParser->yystksz = 1;
}
#endif
#ifndef YYNOERRORRECOVERY
pParser->yyerrcnt = -1;
yypParser->yyerrcnt = -1;
#endif
pParser->yytos = pParser->yystack;
pParser->yystack[0].stateno = 0;
pParser->yystack[0].major = 0;
yypParser->yytos = yypParser->yystack;
yypParser->yystack[0].stateno = 0;
yypParser->yystack[0].major = 0;
#if YYSTACKDEPTH>0
pParser->yystackEnd = &pParser->yystack[YYSTACKDEPTH-1];
yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
#endif
}

Expand All @@ -353,11 +357,14 @@ void ParseInit(void *yypParser){
** A pointer to a parser. This pointer is used in subsequent calls
** to Parse and ParseFree.
*/
void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
yyParser *pParser;
pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
if( pParser ) ParseInit(pParser);
return pParser;
void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){
yyParser *yypParser;
yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
if( yypParser ){
ParseCTX_STORE
ParseInit(yypParser ParseCTX_PARAM);
}
return (void*)yypParser;
}
#endif /* Parse_ENGINEALWAYSONSTACK */

Expand All @@ -374,7 +381,8 @@ static void yy_destructor(
YYCODETYPE yymajor, /* Type code for object to destroy */
YYMINORTYPE *yypminor /* The object to be destroyed */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
switch( yymajor ){
/* Here is inserted the actions which take place when a
** terminal or non-terminal is destroyed. This can happen
Expand Down Expand Up @@ -596,7 +604,8 @@ static int yy_find_reduce_action(
** The following routine is called if the stack overflows.
*/
static void yyStackOverflow(yyParser *yypParser){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
Expand All @@ -608,7 +617,8 @@ static void yyStackOverflow(yyParser *yypParser){
/******** Begin %stack_overflow code ******************************************/
%%
/******** End %stack_overflow code ********************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
ParseARG_STORE /* Suppress warning about unused %extra_argument var */
ParseCTX_STORE
}

/*
Expand Down Expand Up @@ -701,12 +711,13 @@ static void yy_reduce(
unsigned int yyruleno, /* Number of the rule by which to reduce */
int yyLookahead, /* Lookahead token, or YYNOCODE if none */
ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */
ParseCTX_PDECL /* %extra_context */
){
int yygoto; /* The next state */
int yyact; /* The next action */
yyStackEntry *yymsp; /* The top of the parser's stack */
int yysize; /* Amount to pop the stack */
ParseARG_FETCH;
ParseARG_FETCH
(void)yyLookahead;
(void)yyLookaheadToken;
yymsp = yypParser->yytos;
Expand Down Expand Up @@ -789,7 +800,8 @@ static void yy_reduce(
static void yy_parse_failed(
yyParser *yypParser /* The parser */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
Expand All @@ -801,7 +813,8 @@ static void yy_parse_failed(
/************ Begin %parse_failure code ***************************************/
%%
/************ End %parse_failure code *****************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
ParseCTX_STORE
}
#endif /* YYNOERRORRECOVERY */

Expand All @@ -813,12 +826,14 @@ static void yy_syntax_error(
int yymajor, /* The major type of the error token */
ParseTOKENTYPE yyminor /* The minor type of the error token */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#define TOKEN yyminor
/************ Begin %syntax_error code ****************************************/
%%
/************ End %syntax_error code ******************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
ParseCTX_STORE
}

/*
Expand All @@ -827,7 +842,8 @@ static void yy_syntax_error(
static void yy_accept(
yyParser *yypParser /* The parser */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
Expand All @@ -842,7 +858,8 @@ static void yy_accept(
/*********** Begin %parse_accept code *****************************************/
%%
/*********** End %parse_accept code *******************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
ParseCTX_STORE
}

/* The main parser program.
Expand Down Expand Up @@ -878,14 +895,14 @@ void Parse(
#ifdef YYERRORSYMBOL
int yyerrorhit = 0; /* True if yymajor has invoked an error */
#endif
yyParser *yypParser; /* The parser */
yyParser *yypParser = (yyParser*)yyp; /* The parser */
ParseCTX_FETCH
ParseARG_STORE

yypParser = (yyParser*)yyp;
assert( yypParser->yytos!=0 );
#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
yyendofinput = (yymajor==0);
#endif
ParseARG_STORE;

#ifndef NDEBUG
if( yyTraceFILE ){
Expand All @@ -903,7 +920,7 @@ void Parse(
do{
yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
if( yyact >= YY_MIN_REDUCE ){
yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor);
yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor ParseCTX_PARAM);
}else if( yyact <= YY_MAX_SHIFTREDUCE ){
yy_shift(yypParser,yyact,yymajor,yyminor);
#ifndef YYNOERRORRECOVERY
Expand Down

0 comments on commit 2ead4b6

Please sign in to comment.