I want make a parser in happy for the let-in-expression language. For example, i want parse the following string:
let x = 4 in x*x
At the university we study attribute grammars, and i want use this tricks to calculate directly the value of the parsed let-in-expression. So in the happy file, i set the data type of the parsing function to Int, and i created a new attribute called env. This attribute is a function from String to Int that associates variable name to value. Referring to my example:
env "x" = 4
Now i put here below the happy file, where there is my grammar:
{
module Parser where
import Token
import Lexer
}
%tokentype { Token }
%token
let { TLet }
in { TIn }
int { TInt $$ }
var { TVar $$ }
'=' { TEq }
'+' { TPlus }
'-' { TMinus }
'*' { TMul }
'/' { TDiv }
'(' { TOB }
')' { TCB }
%name parse
%attributetype { Int }
%attribute env { String -> Int }
%error { parseError }
%%
Exp : let var '=' Exp in Exp
{
$4.env = $$.env;
$2.env = (\_ -> 0);
$6.env = (\str -> if str == $2 then $4 else 0);
$$ = $6;
}
| Exp1
{
$1.env = $$.env;
$$ = $1;
}
Exp1 : Exp1 '+' Term
{
$1.env = $$.env;
$2.env = $$.env;
$$ = $1 + $3;
}
| Exp1 '-' Term
{
$1.env = $$.env;
$2.env = $$.env;
$$ = $1 - $3;
}
| Term
{
$1.env = $$.env;
$$ = $1;
}
Term : Term '*' Factor
{
$1.env = $$.env;
$2.env = $$.env;
$$ = $1 * $3;
}
| Term '/' Factor
{
$1.env = $$.env;
$2.env = $$.env;
$$ = div $1 $3;
}
| Factor
{
$1.env = $$.env;
$$ = $1;
}
Factor
: int
{
$$ = $1;
}
| var
{
$$ = $$.env $1;
}
| '(' Exp ')'
{
$1.env = $$.env;
$$ = $1;
}
{
parseError :: [Token] -> a
parseError _ = error "Parse error"
}
When i load the haskell file generated from the happy file above, i get the following error:
Ambiguous occurrence `Int'
It could refer to either `Parser.Int', defined at parser.hs:271:6
or `Prelude.Int',
imported from `Prelude' at parser.hs:2:8-13
(and originally defined in `GHC.Types')
I don't know why i get this, because i don't define the type Parser.Int in my happy file. I tried to replace Int with Prelude.Int, but i get other errors.
How can i resolve? Can i have also some general tips if I'm doing something not optimal?
See the happy explaination of attributetype: http://www.haskell.org/happy/doc/html/sec-AtrributeGrammarsInHappy.html
Your line:
%attributetype { Int }
Is declaring a type named Int. This is what causes the ambiguity.
Related
So I am having this .y file on which I am trying to parse and evaluate a function with it's parameters, but a have one shift/reduce conflict that I cannot identify:
.y
%{
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include "types.h"
#define YYDEBUG 0
/* prototypes */
nodeType *opr(int oper, int nops, ...);
nodeType *id(int i);
nodeType *con(int value);
void freeNode(nodeType *p);
void yyerror(char *s);
nodeType *RadEc;
int sym[26]; /* symbol table */
%}
%union {
int iValue; /* integer value */
char sIndex; /* symbol table index */
nodeType *nPtr; /* node pointer */
};
%token <iValue> INTEGER
%token <sIndex> VARIABLE
%token WHILE IF PRINT SUBRAD ENDSUB THEN DO ENDIF RAD
%nonassoc IFX
%nonassoc ELSE
%left GE LE EQ NE '>' '<'
%left '+' '-'
%left '*' '/'
%nonassoc UMINUS
%type <nPtr> statement expr stmt_list
%type <iValue> expresie
%start program
%%
program : declaratii cod { exit(0); }
;
declaratii: SUBRAD stmt_list ENDSUB { RadEc=$2; }
| /* NULL */
;
statement : '\n' { $$ = opr(';', 2, NULL, NULL); }
| expr '\n' { $$ = $1; }
| PRINT expr '\n' { $$ = opr(PRINT, 1, $2); }
| VARIABLE '=' expr '\n' { $$ = opr('=', 2, id($1), $3); }
| DO stmt_list WHILE expr { $$ = opr(WHILE, 2, $4, $2); }
| IF expr THEN stmt_list ENDIF %prec IFX { $$ = opr(IF, 2, $2, $4); }
| IF expr THEN stmt_list ELSE stmt_list ENDIF { $$ = opr(IF, 3, $2, $4, $6); }
;
stmt_list : statement
| stmt_list statement { $$ = opr(';', 2, $1, $2); }
;
expr : INTEGER { $$ = con($1); }
| VARIABLE { $$ = id($1); }
| '-' expr %prec UMINUS { $$ = opr(UMINUS, 1, $2); }
| expr '+' expr { $$ = opr('+', 2, $1, $3); }
| expr '-' expr { $$ = opr('-', 2, $1, $3); }
| expr '*' expr { $$ = opr('*', 2, $1, $3); }
| expr '/' expr { $$ = opr('/', 2, $1, $3); }
| expr '<' expr { $$ = opr('<', 2, $1, $3); }
| expr '>' expr { $$ = opr('>', 2, $1, $3); }
| expr GE expr { $$ = opr(GE, 2, $1, $3); }
| expr LE expr { $$ = opr(LE, 2, $1, $3); }
| expr NE expr { $$ = opr(NE, 2, $1, $3); }
| expr EQ expr { $$ = opr(EQ, 2, $1, $3); }
| '(' expr ')' { $$ = $2; }
;
cod : '.' {exit(0);}
| instruc '\n' cod
;
instruc : '\n'
| PRINT expresie {printf("%d\n",$2);}
| VARIABLE '=' expresie {sym[$1]=$3;}
| RAD'('expresie','expresie','expresie')' {sym[0]=$3; sym[1]=$5; sym[2]=$7; ex(RadEc);}
;
expresie : INTEGER { $$ = $1; }
| VARIABLE { $$ = sym[$1]; }
| '-' expresie %prec UMINUS { $$ = -$2; }
| expresie '+' expresie { $$ = $1+$3; }
| expresie '-' expresie { $$ = $1-$3; }
| expresie '*' expresie { $$ = $1*$3; }
| expresie '/' expresie { $$ = $1/$3; }
| expresie '<' expresie { $$ = $1<$3; }
| expresie '>' expresie { $$ = $1>$3; }
| expresie GE expresie { $$ = $1>=$3; }
| expresie LE expresie { $$ = $1<=$3; }
| expresie NE expresie { $$ = $1!=$3; }
| expresie EQ expresie { $$ = $1==$3; }
| '(' expresie ')' { $$ = $2; }
;
%%
nodeType *con(int value)
{
nodeType *p;
/* allocate node */
if ((p = malloc(sizeof(conNodeType))) == NULL)
yyerror("out of memory");
/* copy information */
p->type = typeCon;
p->con.value = value;
return p;
}
nodeType *id(int i)
{
nodeType *p;
/* allocate node */
if ((p = malloc(sizeof(idNodeType))) == NULL)
yyerror("out of memory");
/* copy information */
p->type = typeId;
p->id.i = i;
return p;
}
nodeType *opr(int oper, int nops, ...)
{
va_list ap;
nodeType *p;
size_t size;
int i;
/* allocate node */
size = sizeof(oprNodeType) + (nops - 1) * sizeof(nodeType*);
if ((p = malloc(size)) == NULL)
yyerror("out of memory");
/* copy information */
p->type = typeOpr;
p->opr.oper = oper;
p->opr.nops = nops;
va_start(ap, nops);
for (i = 0; i < nops; i++)
p->opr.op[i] = va_arg(ap, nodeType*);
va_end(ap);
return p;
}
void freeNode(nodeType *p)
{
int i;
if (!p)
return;
if (p->type == typeOpr) {
for (i = 0; i < p->opr.nops; i++)
freeNode(p->opr.op[i]);
}
free (p);
}
int ex(nodeType *p)
{
if (!p)
return 0;
switch(p->type)
{
case typeCon: return p->con.value;
case typeId: return sym[p->id.i];
case typeOpr: switch(p->opr.oper)
{
case WHILE: while(ex(p->opr.op[0]))
ex(p->opr.op[1]);
return 0;
case IF: if (ex(p->opr.op[0]))
ex(p->opr.op[1]);
else if (p->opr.nops > 2)
ex(p->opr.op[2]);
return 0;
case PRINT: printf("%d\n", ex(p->opr.op[0]));
return 0;
case ';': ex(p->opr.op[0]);
return ex(p->opr.op[1]);
case '=': return sym[p->opr.op[0]->id.i] = ex(p->opr.op[1]);
case UMINUS: return -ex(p->opr.op[0]);
case '+': return ex(p->opr.op[0]) + ex(p->opr.op[1]);
case '-': return ex(p->opr.op[0]) - ex(p->opr.op[1]);
case '*': return ex(p->opr.op[0]) * ex(p->opr.op[1]);
case '/': return ex(p->opr.op[0]) / ex(p->opr.op[1]);
case '<': return ex(p->opr.op[0]) < ex(p->opr.op[1]);
case '>': return ex(p->opr.op[0]) > ex(p->opr.op[1]);
case GE: return ex(p->opr.op[0]) >= ex(p->opr.op[1]);
case LE: return ex(p->opr.op[0]) <= ex(p->opr.op[1]);
case NE: return ex(p->opr.op[0]) != ex(p->opr.op[1]);
case EQ: return ex(p->opr.op[0]) == ex(p->opr.op[1]);
}
}
}
void yyerror(char *s)
{
fprintf(stdout, "%s\n", s);
}
int main(void)
{
#if YYDEBUG
yydebug = 1;
#endif
yyparse();
return 0;
}
I tried different ways to see were am I losing something, but I am pretty new at this and still cannot figure it out very well the conflicts.
Any help much appreciated.
Your grammar allows statements to be expressions and it allows two statements to appear in sequence without any separator.
Now, both of the following are expressions:
a
-1
Suppose they appear like that in a statement list. How is that different from this single expression?
a - 1
Ambiguity always shows up as a parsing conflict.
By the way, delimited if statements (with an endif marker) cannot exhibit the dangling else ambiguity. The endif bracket makes the parse unambiguous. So all of the precedence apparatus copied from a different grammar is totally redundant here.
IHello!
I'm trying to code a lexer and parser for computer algebric system.
When I'm compiling the code with my makefile, I have problems on the value of some functions.
Here is the code of function.ml :
(****************************************
* Type definitions
****************************************)
type operator = Plus | Minus | Times | Div | Power;;
type var = string;; (* e.g. "x", "y", etc. *)
type power = var * float;; (* var, raised_to, e.g. ("x", 3.) -> x^3. *)
type monomial = float * power list;; (* coefficient, list of power terms, e.g. (2., [("x", 1.); ("y", 3.)]) -> 2xy^3 *)
type polynomial = monomial list;; (* sum of monomials, e.g. [(2., [("x", 2.); ("y", 3.)]); (-1., [])] -> 2x^2y^3 - 1 *)
type frac = polynomial * polynomial;; (* numerator, denominator *)
type exp = frac * frac;; (* base, exponent *)
type term = Poly of polynomial
| Frac of frac
| Exp of exp;;
type expr = Leaf of term
| Node of operator * expr list;;
type eqn = expr * expr;;
(****************************************
* Lexer/Parser definitions
****************************************)
type token = PLUS | MINUS | TIMES | DIV | POWER | LPAREN | RPAREN | EQUALS
| FLOAT of float
| VAR of var
| EOF;;
Here is the code of lexer.mll :
{
open Function
}
let numeric = ['0' - '9']
let letter = ['a' - 'z' 'A' - 'Z']
rule main = parse
| [' ' '\t' '\n'] { main lexbuf } (* skip over whitespace *)
| "+" { PLUS }
| "-" { MINUS }
| "*" { TIMES }
| "/" { DIV }
| "^" { POWER }
| "(" { LPAREN }
| ")" { RPAREN }
| "=" { EQUALS }
| ((numeric*) '.' (numeric+)) as n
{ FLOAT (float_of_string n) }
| (numeric+) as n
{ FLOAT (float_of_string n) }
| (letter numeric*) as n
{ VAR n }
| eof { EOF }
{
let lexer_main = main;;
let token_iterator_of_string s =
let rec lbuf = Lexing.from_string s
in fun () -> lexer_main lbuf;;
let token_list_of_string s =
let rec lbuf = Lexing.from_string s
and token_list_aux () =
let token = lexer_main lbuf in
if token = EOF then
[]
else
token :: token_list_aux ()
in token_list_aux ();;
}
Here is the code of parser.mly :
%{
open Function
%}
%token PLUS MINUS TIMES DIV POWER LPAREN RPAREN EQUALS EOF
%token <float> FLOAT
%token <Function.var> VAR
%start yacc_eqn
%start yacc_expr
%type <Function.eqn> yacc_eqn
%type <Function.expr> yacc_expr
%%
yacc_eqn:
exp EQUALS exp EOF { ($1, $3) }
yacc_expr:
exp EOF { $1 }
exp:
op1 { $1 }
op1:
op2 { $1 }
| op1 PLUS op2 { Node(Plus, [$1; $3]) }
| op1 MINUS op2 { Node(Minus, [$1; $3]) }
op2:
op3 { $1 }
| op2 TIMES op3 { Node(Times, [$1; $3]) }
| op2 DIV op3 { Node(Div, [$1; $3]) }
op3:
op4 { $1 }
| op3 op4 { Node(Times, [$1; $2]) }
op4:
leaf { $1 }
| op4 POWER leaf { Node(Power, [$1; $3]) }
leaf:
atom { $1 }
| LPAREN exp RPAREN { $2 }
atom:
VAR { Leaf(Poly(poly_of_var $1)) }
| FLOAT { Leaf(Poly(poly_of_float $1)) }
%%
let eqn_of_string s = yacc_eqn Lexer.lexer_main (Lexing.from_string s);;
let expr_of_string s = yacc_expr Lexer.lexer_main (Lexing.from_string s);;
let parse_eqn = eqn_of_string;;
let parse_expr = expr_of_string;;
The problem is that in the parser.mli which is create with the makefile, the value of yacc_eqn and yacc_expr are :
val yacc_eqn :
(Lexing.lexbuf -> token) -> Lexing.lexbuf -> Function.eqn
val yacc_expr :
(Lexing.lexbuf -> token) -> Lexing.lexbuf -> Function.expr
And I have the following error :
The implementation parser.ml does not match the interface parser.cmi:
Values do not match:
val yacc_eqn :
(Lexing.lexbuf -> Function.token) -> Lexing.lexbuf -> Function.eqn
is not included in
val yacc_eqn :
(Lexing.lexbuf -> token) -> Lexing.lexbuf -> Function.eqn
I think the solution might be to something like a cast, but I have stricly no idea how to do that ... Anyone help ?
Thanks in advance !
This is not homework, but it is from a book.
I'm given a following bison spec file:
%{
#include <stdio.h>
#include <ctype.h>
int yylex();
int yyerror();
%}
%token NUMBER
%%
command : exp { printf("%d\n", $1); }
; /* allows printing of the result */
exp : exp '+' term { $$ = $1 + $3; }
| exp '-' term { $$ = $1 - $3; }
| term { $$ = $1; }
;
term : term '*' factor { $$ = $1 * $3; }
| factor { $$ = $1; }
;
factor : NUMBER { $$ = $1; }
| '(' exp ')' { $$ = $2; }
;
%%
int main() {
return yyparse();
}
int yylex() {
int c;
/* eliminate blanks*/
while((c = getchar()) == ' ');
if (isdigit(c)) {
ungetc(c, stdin);
scanf("%d", &yylval);
return (NUMBER);
}
/* makes the parse stop */
if (c == '\n') return 0;
return (c);
}
int yyerror(char * s) {
fprintf(stderr, "%s\n", s);
return 0;
} /* allows for printing of an error message */
The task is to do the following:
Rewrite the spec to add the following useful error messages:
"missing right parenthesis," generated by the string (2+3
"missing left parenthesis," generated by the string 2+3)
"missing operator," generated by the string 2 3
"missing operand," generated by the string (2+)
The simplest solution that I was able to come up with is to do the following:
half_exp : exp '+' { $$ = $1; }
| exp '-' { $$ = $1; }
| exp '*' { $$ = $1; }
;
factor : NUMBER { $$ = $1; }
| '(' exp '\n' { yyerror("missing right parenthesis"); }
| exp ')' { yyerror("missing left parenthesis"); }
| '(' exp '\n' { yyerror("missing left parenthesis"); }
| '(' exp ')' { $$ = $2; }
| '(' half_exp ')' { yyerror("missing operand"); exit(0); }
;
exp : exp '+' term { $$ = $1 + $3; }
| exp '-' term { $$ = $1 - $3; }
| term { $$ = $1; }
| exp exp { yyerror("missing operator"); }
;
These changes work, however they lead to a lot of conflicts.
Here is my question.
Is there a way to rewrite this grammar in such a way so that it wouldn't generate conflicts?
Any help is appreciated.
Yes it is possible:
command : exp { printf("%d\n", $1); }
; /* allows printing of the result */
exp: exp '+' exp {
// code
}
| exp '-' exp {
// code
}
| exp '*' exp {
// code
}
| exp '/' exp {
// code
}
|'(' exp ')' {
// code
}
Bison allows Ambiguous grammars.
I don't see how can you rewrite grammar to avoid conflicts. You just missed the point of terms, factors etc. You use these when you want left recursion context free grammar.
From this grammar:
E -> E+T
|T
T -> T*F
|F
F -> (E)
|num
Once you free it from left recursion you would go to:
E -> TE' { num , ( }
E' -> +TE' { + }
| eps { ) , EOI }
T -> FT' { ( , num }
T' -> *FT' { * }
|eps { + , ) , EOI }
F -> (E) { ( }
|num { num }
These sets alongside rules are showing what input character has to be in order to use that rule. Of course this is just example for simple arithmetic expressions for example 2*(3+4)*5+(3*3*3+4+5*6) etc.
If you want to learn more about this topic I suggest you to read about "left recursion context free grammar". There are some great books covering this topic and also covering how to get input sets.
But as I said above, all of this can be avoided because Bison allows Ambiguous grammars.
I'm trying to get familiar with Happy parser generator for Haskell. Currently, I have an example from the documentation but when I compile the program, I get an error.
This is the code:
{
module Main where
import Data.Char
}
%name calc
%tokentype { Token }
%error { parseError }
%token
let { TokenLet }
in { TokenIn }
int { TokenInt $$ }
var { TokenVar $$ }
'=' { TokenEq }
'+' { TokenPlus }
'-' { TokenMinus }
'*' { TokenTimes }
'/' { TokenDiv }
'(' { TokenOB }
')' { TokenCB }
%%
Exp : let var '=' Exp in Exp { \p -> $6 (($2,$4 p):p) }
| Exp1 { $1 }
Exp1 : Exp1 '+' Term { \p -> $1 p + $3 p }
| Exp1 '-' Term { \p -> $1 p - $3 p }
| Term { $1 }
Term : Term '*' Factor { \p -> $1 p * $3 p }
| Term '/' Factor { \p -> $1 p `div` $3 p }
| Factor { $1 }
Factor
: int { \p -> $1 }
| var { \p -> case lookup $1 p of
Nothing -> error "no var"
Just i -> i }
| '(' Exp ')' { $2 }
{
parseError :: [Token] -> a
parseError _ = error "Parse error"
data Token
= TokenLet
| TokenIn
| TokenInt Int
| TokenVar String
| TokenEq
| TokenPlus
| TokenMinus
| TokenTimes
| TokenDiv
| TokenOB
| TokenCB
deriving Show
lexer :: String -> [Token]
lexer [] = []
lexer (c:cs)
| isSpace c = lexer cs
| isAlpha c = lexVar (c:cs)
| isDigit c = lexNum (c:cs)
lexer ('=':cs) = TokenEq : lexer cs
lexer ('+':cs) = TokenPlus : lexer cs
lexer ('-':cs) = TokenMinus : lexer cs
lexer ('*':cs) = TokenTimes : lexer cs
lexer ('/':cs) = TokenDiv : lexer cs
lexer ('(':cs) = TokenOB : lexer cs
lexer (')':cs) = TokenCB : lexer cs
lexNum cs = TokenInt (read num) : lexer rest
where (num,rest) = span isDigit cs
lexVar cs =
case span isAlpha cs of
("let",rest) -> TokenLet : lexer rest
("in",rest) -> TokenIn : lexer rest
(var,rest) -> TokenVar var : lexer rest
main = getContents >>= print . calc . lexer
}
I'm getting this error:
[1 of 1] Compiling Main ( gr.hs, gr.o )
gr.hs:310:24:
No instance for (Show ([(String, Int)] -> Int))
arising from a use of `print'
Possible fix:
add an instance declaration for (Show ([(String, Int)] -> Int))
In the first argument of `(.)', namely `print'
In the second argument of `(>>=)', namely `print . calc . lexer'
In the expression: getContents >>= print . calc . lexer
Do you know why and how can I solve it?
If you examine the error message
No instance for (Show ([(String, Int)] -> Int))
arising from a use of `print'
it's clear that the problem is that you are trying to print a function. And indeed, the value produced by the parser function calc is supposed to be a function which takes a lookup table of variable bindings and gives back a result. See for example the rule for variables:
{ \p -> case lookup $1 p of
Nothing -> error "no var"
Just i -> i }
So in main, we need to pass in a list for the p argument, for example an empty list. (Or you could add some pre-defined global variables if you wanted). I've expanded the point-free code to a do block so it's easier to see what's going on:
main = do
input <- getContents
let fn = calc $ lexer input
print $ fn [] -- or e.g. [("foo", 42)] if you wanted it pre-defined
Now it works:
$ happy Calc.y
$ runghc Calc.hs <<< "let x = 1337 in x * 2"
2674
Type mismatch. Expecting a LexBuffer<char> but given a LexBuffer<byte> The type 'char' does not match the type 'byte'
This is the error message that I am getting while using fslex. I have tried manually checking every single occurrence of lexbuf and its type. It's LexBuffer<char> everywhere. But still the compiler is giving me the above error. Can you please tell me why this error occurs and how to go about resolving it.
{
open System
open Microsoft.FSharp.Text.Lexing
open Microsoft.FSharp.Text.Parsing
let lexeme (lexbuf : LexBuffer<char>) = new System.String(lexbuf.Lexeme)
let newline (lexbuf:LexBuffer<char>) = lexbuf.EndPos <- lexbuf.EndPos.NextLine
let unexpected_char (lexbuf:LexBuffer<char>) = failwith ("Unexpected character '"+(lexeme lexbuf)+"'")
}
let char = ['a'-'z' 'A'-'Z']
let digit = ['0'-'9']
let float = '-'?digit+ '.' digit+
let ident = char+ (char | digit)*
let whitespace = [' ' '\t']
let newline = ('\n' | '\r' '\n')
rule tokenize = parse
| "maximize" { MAXIMIZE }
| "minimize" { MINIMIZE }
| "where" { WHERE }
| '+' { PLUS }
| '-' { MINUS }
| '*' { MULTIPLY }
| '=' { EQUALS }
| '>' { STRICTGREATERTHAN }
| '<' { STRICTLESSTHAN }
| ">=" { GREATERTHANEQUALS }
| "<=" { LESSTHANEQUALS }
| '[' { LSQUARE }
| ']' { RSQUARE }
| whitespace { tokenize lexbuf }
| newline { newline lexbuf; tokenize lexbuf }
| ident { ID (lexeme lexbuf) }
| float { FLOAT (Double.Parse(lexeme lexbuf)) }
| ';' { SEMICOLON }
| eof { EOF }
| _ { unexpected_char lexbuf }
Maybe you need to generate a unicode lexer. A unicode lexer works with a LexBuffer<char> rather than LexBuffer<byte>.
The "unicode" argument to FsLex is optional, but if enabled generates a unicode lexer.
http://blogs.msdn.com/dsyme/archive/2009/10/21/some-smaller-features-in-the-latest-release-of-f.aspx
Have you tried inserting an explicit cast?
There was a mistake with my lexer file definition I believe, it compiled when I made the following my lexer definition. Experts can throw more insight into the reasons, while the understanding that I have is the type of the lexbuf that is used in the lexer should somehow be related to the definition that the parser generates
{
open System
open LanguageParser
open Microsoft.FSharp.Text.Lexing
open Microsoft.FSharp.Text.Parsing
open System.Text
let newline (lexbuf:LexBuffer<_>) = lexbuf.EndPos <- lexbuf.EndPos.NextLine
}
let char = ['a'-'z' 'A'-'Z']
let digit = ['0'-'9']
let float = '-'?digit+ '.' digit+
let ident = char+ (char | digit)*
let whitespace = [' ' '\t']
let newline = ('\n' | '\r' '\n')
rule tokenize = parse
| "maximize" { MAXIMIZE }
| "minimize" { MINIMIZE }
| "where" { WHERE }
| '+' { PLUS }
| '-' { MINUS }
| '*' { MULTIPLY }
| '=' { EQUALS }
| '>' { STRICTGREATERTHAN }
| '<' { STRICTLESSTHAN }
| ">=" { GREATERTHANEQUALS }
| "<=" { LESSTHANEQUALS }
| '[' { LSQUARE }
| ']' { RSQUARE }
| whitespace { tokenize lexbuf }
| newline { newline lexbuf; tokenize lexbuf }
| ident { ID <| Encoding.UTF8.GetString(lexbuf.Lexeme) }
| float { FLOAT <| Double.Parse(Encoding.UTF8.GetString(lexbuf.Lexeme)) }
| ';' { SEMICOLON }
| eof { EOF }
| _ { failwith ("Unexpected Character") }