Does YACC (LALR) parses left recursive grammars - parsing

I was doing a simple C parser project.
This problem occurred when I was writing the grammar for if-else construct.
the grammar that I have written is as following:
iexp: IF OP exp CP block eixp END{
printf("Valid if-else ladder\n");
};
eixp:
|ELSE iexp
|ELSE block;
exp: id
|NUMBER
|exp EQU exp
|exp LESS exp
|exp GRT exp
|OP exp CP;
block: statement
|iexp
|OCP statement CCP
|OCP iexp CCP;
statement: id ASSIGN NUMBER SEMICOL;
id: VAR;
where the lex part looks something like this
"if" {return IF;}
"else" {return ELSE;}
[0-9]+ {return NUMBER;}
">" {return GRT;}
"<" {return LESS;}
"==" {return EQU;}
"{" {return OCP;}
"}" {return CCP;}
"(" {return OP;}
")" {return CP;}
"$" {return END;}
";" {return SEMICOL;}
"=" {return ASSIGN;}
[a-zA-Z]+ {return VAR;}
. {;}
I am getting o/p as
yacc: 9 shift/reduce conflicts, 1 reduce/reduce conflict.
When I eliminate the left recursion on exp derivation the conflicts vanish but why it's so ?
the revised grammar after eliminating left recursion was :
exp: id
|NUMBER
|id EQU exp
|id LESS exp
|id GRT exp
|OP exp CP;
I was able to parse successfully the grammar for the evaluation of arithmetic expressions. Is it so that %right, %left made it successful
%token ID
%left '+' '-'
%left '*' '/'
%right NEGATIVE
%%
S:E {
printf("\nexpression : %s\nResult=%d\n", buf, $$);
buf[0] = '\0';
};
E: E '+' E {
printf("+");
($$ = $1 + $3);
} |
E '-' E {
printf("-");
($$ = $1 - $3);
} |
E '*' E {
printf("*");
($$ = $1 * $3);
} |
E '/' E {
printf("/");
($$ = $1 / $3);
} |
'(' E ')' {
($$ = $2);
} |
ID {
/*do nothing done by lex*/
};

Related

Parser (Yacc) seems like it ignores tokens in grammar

Parsing the c-like example code, i have the following issue. Its like some tokens, like identifiers, are ignored by grammar, causing a non-reason syntax error.
Parser code :
%{
#include <stdio.h>
#include <stdlib.h>
int yylex();
void yyerror (char const *);
%}
%token T_MAINCLASS T_ID T_PUBLIC T_STATIC T_VOID T_MAIN T_PRINTLN T_INT T_FLOAT T_FOR T_WHILE T_IF T_ELSE T_EQUAL T_SMALLER T_BIGGER T_NOTEQUAL T_NUM T_STRING
%left '(' ')'
%left '+' '-'
%left '*' '/'
%left '{' '}'
%left ';' ','
%left '<' '>'
%%
PROGRAM : T_MAINCLASS T_ID '{' T_PUBLIC T_STATIC T_VOID T_MAIN '(' ')' COMP_STMT '}'
;
COMP_STMT : '{' STMT_LIST '}'
;
STMT_LIST : /* nothing */
| STMT_LIST STMT
;
STMT : ASSIGN_STMT
| FOR_STMT
| WHILE_STMT
| IF_STMT
| COMP_STMT
| DECLARATION
| NULL_STMT
| T_PRINTLN '(' EXPR ')' ';'
;
DECLARATION : TYPE ID_LIST ';'
;
TYPE : T_INT
| T_FLOAT
;
ID_LIST : T_ID ',' ID_LIST
|
;
NULL_STMT : ';'
;
ASSIGN_STMT : ASSIGN_EXPR ';'
;
ASSIGN_EXPR : T_ID '=' EXPR
;
EXPR : ASSIGN_EXPR
| RVAL
;
FOR_STMT : T_FOR '(' OPASSIGN_EXPR ';' OPBOOL_EXPR ';' OPASSIGN_EXPR ')' STMT
;
OPASSIGN_EXPR : /* nothing */
| ASSIGN_EXPR
;
OPBOOL_EXPR : /* nothing */
| BOOL_EXPR
;
WHILE_STMT : T_WHILE '(' BOOL_EXPR ')' STMT
;
IF_STMT : T_IF '(' BOOL_EXPR ')' STMT ELSE_PART
;
ELSE_PART : /* nothing */
| T_ELSE STMT
;
BOOL_EXPR : EXPR C_OP EXPR
;
C_OP : T_EQUAL | '<' | '>' | T_SMALLER | T_BIGGER | T_NOTEQUAL
;
RVAL : RVAL '+' TERM
| RVAL '-' TERM
| TERM
;
TERM : TERM '*' FACTOR
| TERM '/' FACTOR
| FACTOR
;
FACTOR : '(' EXPR ')'
| T_ID
| T_NUM
;
%%
void yyerror (const char * msg)
{
fprintf(stderr, "C-like : %s\n", msg);
exit(1);
}
int main ()
{
if(!yyparse()){
printf("Compiled !!!\n");
}
}
Part of Lexical Scanner code :
{Empty}+ { printf("EMPTY ") ; /* nothing */ }
"mainclass" { printf("MAINCLASS ") ; return T_MAINCLASS ; }
"public" { printf("PUBLIC ") ; return T_PUBLIC; }
"static" { printf("STATIC ") ; return T_STATIC ; }
"void" { printf("VOID ") ; return T_VOID ; }
"main" { printf("MAIN ") ; return T_MAIN ; }
"println" { printf("PRINTLN ") ; return T_PRINTLN ; }
"int" { printf("INT ") ; return T_INT ; }
"float" { printf("FLOAT ") ; return T_FLOAT ; }
"for" { printf("FOR ") ; return T_FOR ; }
"while" { printf("WHILE ") ; return T_WHILE ; }
"if" { printf("IF ") ; return T_IF ; }
"else" { printf("ELSE ") ; return T_ELSE ; }
"==" { printf("EQUAL ") ; return T_EQUAL ; }
"<=" { printf("SMALLER ") ; return T_SMALLER ; }
">=" { printf("BIGGER ") ; return T_BIGGER ; }
"!=" { printf("NOTEQUAL ") ; return T_NOTEQUAL ; }
{id} { printf("ID ") ; return T_ID ; }
{num} { printf("NUM ") ; return T_NUM ; }
{string} { printf("STRING ") ; return T_STRING ; }
{punct} { printf("PUNCT ") ; return yytext[0] ; }
<<EOF>> { printf("EOF ") ; return T_EOF; }
. { yyerror("lexical error"); exit(1); }
Example :
mainclass Example {
public static void main ( )
{
int c;
float x, sum, mo;
c=0;
x=3.5;
sum=0.0;
while (c<5)
{
sum=sum+x;
c=c+1;
x=x+1.5;
}
mo=sum/5;
println (mo);
}
}
Running all this stuff it showed up this output:
C-like : syntax error
MAINCLASS EMPTY ID
It seems like id is in wrong position although in grammar we have:
PROGRAM : T_MAINCLASS T_ID '{' T_PUBLIC T_STATIC T_VOID T_MAIN '(' ')' COMP_STMT '}'
Based on the "solution" proposed in OP's self answer, it's pretty clear that the original problem was that the generated header used to compile the scanner was not the same as the header generated by bison/yacc from the parser specification.
The generated header includes definitions of all the token types as small integers; in order for the scanner to communicate with the parser, it must identify each token with the correct token type. So the parser generator (bison/yacc) produces a header based on the parser specification (the .y file), and that header must be #included into the generated scanner so that scanner actions can used symbolic token type names.
If the scanner was compiled with a header file generated from some previous version of the parser specification, it is quite possible that the token numbers no longer correspond with what the parser is expecting.
The easiest way to avoid this problem is to use a build system like make, which will automatically recompile the scanner if necessary.
The easiest way to detect this problem is to use bison's built-in trace facility. Enabling tracing requires only a couple of lines of code, and saves you from having to scatter printf statements throughout your scanner and parser. The bison trace will show you exactly what is going on, so not only is it less work than adding printfs, it is also more precise. In particular, it reports every token which is passed to the parser (and, with a little more effort, you can get it to report the semantic values of those tokens as well). So if the parser is getting the wrong token code, you'll see that right away.
After many potential helpful changes, parser worked by changing the order of these tokens.
From
%token T_MAINCLASS T_ID T_PUBLIC T_STATIC T_VOID T_MAIN T_PRINTLN T_INT T_FLOAT T_FOR T_WHILE T_IF T_ELSE T_EQUAL T_SMALLER T_BIGGER T_NOTEQUAL T_NUM T_STRING
TO
%token T_MAINCLASS T_PUBLIC T_STATIC T_VOID T_MAIN T_PRINTLN T_INT T_FLOAT T_FOR T_WHILE T_IF T_EQUAL T_ID T_NUM T_SMALLER T_BIGGER T_NOTEQUAL T_ELSE T_STRING
It looked like that the reading element was else but lexer normaly returned an id. Somehow this modification was the solution.

conflicts: 2 shift/reduce

I'm trying to write a little interpreter with GNU bison.
I wanted to ask if anyone could explain the difference between the directive% right and% left and where my mistake is in the code below.
%token <flo> FLO
%token <name> NAME
%right '='
%left '+' '-'
%left '*' '/' '%'
%left '&' '|' 'x'
%left NEG NOT LOGIC_NOT
%left '^'
%left ARG
%type <flo> exp
%%
language: /* nothing */
| language statment
statment: '\n'
| exp
| error { yyerrok; }
;
exp: FLO { $$ = $1; }
| NAME '(' ')' { $$ = ycall($1); }
| NAME '(' exp ')' { $$ = ycall($1, $3); }
| NAME '(' exp ',' exp ')' { $$ = ycall($1, $3, $5); }
| NAME '=' exp { $$ = 1; ysetvar($1, $3); }
| NAME %prec VAR { $$ = ygetvar($1); }
| '_' exp %prec ARG { $$ = ygetarg($2, args); }
| '(' exp ')' { $$ = $2; }
/* 1 Operand */
| '-' exp %prec NEG { $$ = - $2; }
| '~' exp %prec NOT { $$ = ~ static_cast<int>($2); }
| '!' exp %prec LOGIC_NOT { $$ = ! static_cast<int>($2); }
/* 2 Operands */
| exp '+' exp { $$ = $1 + $3; }
| exp '-' exp { $$ = $1 - $3; }
| exp '*' exp { $$ = $1 * $3; }
| exp '/' exp { $$ = $1 / $3; }
| exp '%' exp { $$ = static_cast<int>($1) % static_cast<int>($3); }
| exp '^' exp { $$ = pow($1, $3); }
| exp '&' exp { $$ = static_cast<int>($1) & static_cast<int>($3); }
| exp '|' exp { $$ = static_cast<int>($1) | static_cast<int>($3); }
| exp 'x' exp { $$ = static_cast<int>($1) ^ static_cast<int>($3); }
;
Look at the y.output file produced by yacc or bison with the -v argument. The first conflict is in state 5:
State 5
7 exp: NAME . '(' ')'
8 | NAME . '(' exp ')'
9 | NAME . '(' exp ',' exp ')'
10 | NAME . '=' exp
11 | NAME .
'=' shift, and go to state 14
'(' shift, and go to state 15
'(' [reduce using rule 11 (exp)]
$default reduce using rule 11 (exp)
In this case the conflcit is when there's a '(' after a NAME -- this is an ambiguity in your grammar in which it might be a call expression, or it might be a simple NAME expression followed by a parenthesized expression, due to the fact that you have no separator between statements in your language.
The second conflict is:
State 13
4 statment: exp .
17 exp: exp . '+' exp
18 | exp . '-' exp
19 | exp . '*' exp
20 | exp . '/' exp
21 | exp . '%' exp
22 | exp . '^' exp
23 | exp . '&' exp
24 | exp . '|' exp
25 | exp . 'x' exp
'+' shift, and go to state 21
'-' shift, and go to state 22
'*' shift, and go to state 23
'/' shift, and go to state 24
'%' shift, and go to state 25
'&' shift, and go to state 26
'|' shift, and go to state 27
'x' shift, and go to state 28
'^' shift, and go to state 29
'-' [reduce using rule 4 (statment)]
$default reduce using rule 4 (statment)
which is essentially the same problem, this time with a '-' -- the input NAME - NAME might be a single binary subtract statements, or it might be two statements -- a NAME followed by a unary negate.
If you add a separator between statements (such as ;), both of these conflicts would go away.

unclear how to add extra productions to bison grammar to create error messages

This is not homework, but it is from a book.
I'm given a following bison spec file:
%{
#include <stdio.h>
#include <ctype.h>
int yylex();
int yyerror();
%}
%token NUMBER
%%
command : exp { printf("%d\n", $1); }
; /* allows printing of the result */
exp : exp '+' term { $$ = $1 + $3; }
| exp '-' term { $$ = $1 - $3; }
| term { $$ = $1; }
;
term : term '*' factor { $$ = $1 * $3; }
| factor { $$ = $1; }
;
factor : NUMBER { $$ = $1; }
| '(' exp ')' { $$ = $2; }
;
%%
int main() {
return yyparse();
}
int yylex() {
int c;
/* eliminate blanks*/
while((c = getchar()) == ' ');
if (isdigit(c)) {
ungetc(c, stdin);
scanf("%d", &yylval);
return (NUMBER);
}
/* makes the parse stop */
if (c == '\n') return 0;
return (c);
}
int yyerror(char * s) {
fprintf(stderr, "%s\n", s);
return 0;
} /* allows for printing of an error message */
The task is to do the following:
Rewrite the spec to add the following useful error messages:
"missing right parenthesis," generated by the string (2+3
"missing left parenthesis," generated by the string 2+3)
"missing operator," generated by the string 2 3
"missing operand," generated by the string (2+)
The simplest solution that I was able to come up with is to do the following:
half_exp : exp '+' { $$ = $1; }
| exp '-' { $$ = $1; }
| exp '*' { $$ = $1; }
;
factor : NUMBER { $$ = $1; }
| '(' exp '\n' { yyerror("missing right parenthesis"); }
| exp ')' { yyerror("missing left parenthesis"); }
| '(' exp '\n' { yyerror("missing left parenthesis"); }
| '(' exp ')' { $$ = $2; }
| '(' half_exp ')' { yyerror("missing operand"); exit(0); }
;
exp : exp '+' term { $$ = $1 + $3; }
| exp '-' term { $$ = $1 - $3; }
| term { $$ = $1; }
| exp exp { yyerror("missing operator"); }
;
These changes work, however they lead to a lot of conflicts.
Here is my question.
Is there a way to rewrite this grammar in such a way so that it wouldn't generate conflicts?
Any help is appreciated.
Yes it is possible:
command : exp { printf("%d\n", $1); }
; /* allows printing of the result */
exp: exp '+' exp {
// code
}
| exp '-' exp {
// code
}
| exp '*' exp {
// code
}
| exp '/' exp {
// code
}
|'(' exp ')' {
// code
}
Bison allows Ambiguous grammars.
I don't see how can you rewrite grammar to avoid conflicts. You just missed the point of terms, factors etc. You use these when you want left recursion context free grammar.
From this grammar:
E -> E+T
|T
T -> T*F
|F
F -> (E)
|num
Once you free it from left recursion you would go to:
E -> TE' { num , ( }
E' -> +TE' { + }
| eps { ) , EOI }
T -> FT' { ( , num }
T' -> *FT' { * }
|eps { + , ) , EOI }
F -> (E) { ( }
|num { num }
These sets alongside rules are showing what input character has to be in order to use that rule. Of course this is just example for simple arithmetic expressions for example 2*(3+4)*5+(3*3*3+4+5*6) etc.
If you want to learn more about this topic I suggest you to read about "left recursion context free grammar". There are some great books covering this topic and also covering how to get input sets.
But as I said above, all of this can be avoided because Bison allows Ambiguous grammars.

Bison nonassociative precedence rule

I am trying to enforce some parsing errors by defining a non-associative precedence.
Here is part of my grammar file:
Comparison :
Value ComparisonOp Value
{
$2->Left($1);
$2->Right($3);
$$ = $2;
}
;
Expressions like 1 = 2 should parse, but expressions like 1 = 2 = 3 are not allowed in the grammar. To accommodate this, I tried to make my operator non associative as follows:
%nonassoc NONASSOCIATIVE
.
.(rest of the grammar)
.
Comparison :
Value ComparisonOp Value %prec NONASSOCIATIVE
{
$2->Left($1);
$2->Right($3);
$$ = $2;
}
;
1 = 2 = 3 still passes, Could someone please tell me what I am doing wrong?
You need to set the associativity of the ComparisonOp token(s). Either
%nonassoc ComparisonOp NONASSOCIATIVE
if ComparisonOp is a token or something like
%nonassoc '=' '<' '>' NOT_EQUAL GREATOR_OR_EQUAL LESS_OR_EQUAL NONASSOCIATIVE
if you have mulitple tokens and ComparisonOp is a rule that expands to any of them
As a concrete example, the following works exactly like you are requesting:
%{
#include <stdio.h>
#include <ctype.h>
int yylex();
void yyerror(const char *);
%}
%nonassoc '=' '<' '>' CMPOP
%left '+' '-' ADDOP
%left '*' '/' '%' MULOP
%token VALUE
%%
expr: expr cmp_op expr %prec CMPOP
| expr add_op expr %prec ADDOP
| expr mul_op expr %prec MULOP
| VALUE
| '(' expr ')'
;
cmp_op: '=' | '<' | '>' | '<' '=' | '>' '=' | '<' '>' ;
add_op: '+' | '-' ;
mul_op: '*' | '/' | '%' ;
%%
int main() { return yyparse(); }
int yylex() {
int ch;
while(isspace(ch = getchar()));
if (isdigit(ch)) return VALUE;
return ch;
}
void yyerror(const char *err) { fprintf(stderr, "%s\n", err); }
so if you are having other problems, try posting an MVCE that shows the actual problem you are having...

Attempting to resolve shift-reduce parsing issue

I'm attempting to write a grammar for C and am having an issue that I don't quite understand. Relevant portions of the grammar:
stmt :
types decl SEMI { marks (A.Declare ($1, $2)) (1, 2) }
| simp SEMI { marks $1 (1, 1) }
| RETURN exp SEMI { marks (A.Return $2) (1, 2) }
| control { $1 }
| block { marks $1 (1, 1) }
;
control :
if { $1 }
| WHILE RPAREN exp LPAREN stmt { marks (A.While ($3, $5)) (1, 5) }
| FOR LPAREN simpopt SEMI exp SEMI simpopt RPAREN stmt { marks (A.For ($3, $5, $7, $9)) (1, 9) }
;
if :
IF RPAREN exp LPAREN stmt { marks (A.If ($3, $5, None)) (1, 5) }
| IF RPAREN exp LPAREN stmt ELSE stmt { marks (A.If ($3, $5, $7)) (1, 7) }
;
This doesn't work. I ran ocamlyacc -v and got the following report:
83: shift/reduce conflict (shift 86, reduce 14) on ELSE
state 83
if : IF RPAREN exp LPAREN stmt . (14)
if : IF RPAREN exp LPAREN stmt . ELSE stmt (15)
ELSE shift 86
IF reduce 14
WHILE reduce 14
FOR reduce 14
BOOL reduce 14
IDENT reduce 14
RETURN reduce 14
INT reduce 14
MAIN reduce 14
LBRACE reduce 14
RBRACE reduce 14
LPAREN reduce 14
I've read that shift/reduce conflicts are due to ambiguity in the specification of the grammar, but I don't see how I can specify this in a way that isn't ambiguous?
The grammar is certainly ambiguous, although you know what every string means, and furthermore despite the fact that ocamlyacc reports a shift/reduce conflict, its generated grammar will also produce the correct parse for every valid input.
The ambiguity comes from
if ( exp1 ) if ( exp2) stmt1 else stmt2;
Clearly stmt1 only executes if both exp1 and exp2 are true. But does stmt1 execute if exp1 is false, or if exp1 is true and exp2 is false? Those represent different parses; the first (invalid) parse attaches else stmt2 to if (exp1), while the parse that you, I and ocamlyacc know to be correct attaches else stmt2 to if (exp2).
The grammar can be rewritten, although it's a bit of a nuisance. The basic idea is to divide statements into two categories: "matched" (which means that every else in the statement is matched with some if) and "unmatched" (which means that a following else would match some if in the statement. A complete statement may be unmatched, because else clauses are optional, but you can never have an unmatched statement between an if and an else, because that else must match an if in the unmatched statement.
The following grammar is basically the one you provided, but rewritten to use bison-style single-quoted tokens, which I find more readable. I don't know if ocamlyacc handles those. (By the way, your grammar says IF RPAREN exp LPAREN... which, with the common definition of left and right parentheses, would mean if ) exp (. That's one reason I find single-quoted character terminals much more readable.)
Bison handles this grammar with no conflicts.
/* Fake non-terminals */
%token types decl simp exp
/* Keywords */
%token ELSE FOR IF RETURN WHILE
%%
stmt: matched_stmt | unmatched_stmt ;
stmt_list: stmt | stmt_list stmt ;
block: '{' stmt_list '}' ;
matched_stmt
: types decl ';'
| simp ';'
| RETURN exp ';'
| block
| matched_control
;
simpopt : simp | /* EMPTY */;
matched_control
: IF '(' exp ')' matched_stmt ELSE matched_stmt
| WHILE '(' exp ')' matched_stmt
| FOR '(' simpopt ';' exp ';' simpopt ')' matched_stmt
;
unmatched_stmt
: IF '(' exp ')' stmt
| IF '(' exp ')' matched_stmt ELSE unmatched_stmt
| WHILE '(' exp ')' unmatched_stmt
| FOR '(' simpopt ';' exp ';' simpopt ')' unmatched_stmt
;
Personally, I'd refactor a bit. Eg:
if_prefix : IF '(' exp ')'
;
loop_prefix: WHILE '(' exp ')'
| FOR '(' simpopt ';' exp ';' simpopt ')'
;
matched_control
: if_prefix matched_stmt ELSE matched_stmt
| loop_prefix matched_stmt
;
unmatched_stmt
: if_prefix stmt
| if_prefix ELSE unmatched_stmt
| loop_prefix unmatched_stmt
;
A common and simpler but less rigorous solution is to use precedence declarations as suggested in the bison manual.

Resources