From: Eric S. Raymond Date: Tue, 20 Jan 2004 20:39:32 +0000 (+0000) Subject: Sent to the author. X-Git-Tag: 1.0~180 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=936c19f99ea01fefca1824ea9e2775da55ddff92;p=mad.git Sent to the author. --- diff --git a/lex.py b/lex.py index 06df0b1..e0b2519 100644 --- a/lex.py +++ b/lex.py @@ -8,7 +8,7 @@ # # Copyright (C) 2001, David M. Beazley # -# $Header: /home/esr/src/ply-1.3.1/RCS/lex.py,v 1.1 2004/01/05 21:34:32 esr Exp $ +# $Header: /home/esr/WWW/mad/RCS/lex.py,v 1.1 2004/01/20 17:55:46 esr Exp esr $ # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -154,7 +154,7 @@ if the input looks very corrupted. The lex module defines a t.lineno attribute on each token that can be used to track the current line number in the input. The value of this -variable is not modified by lex so it is up to your lexer module +variable is not normally modified by lex so it is up to your lexer module to correctly update its value depending on the lexical properties of the input language. To do this, you might write rules such as the following: @@ -163,6 +163,9 @@ def t_newline(t): r' \n+ ' t.lineno += t.value.count("\n") +Exception: if you put \n in the ignore set, the lexer's internal line +counter will be incremented each time \n is seen. + To initialize your lexer so that it can be used, simply call the lex.lex() function in your rule file. If there are any errors in your specification, warning messages or an exception will be generated to @@ -287,6 +290,8 @@ class Lexer: while lexpos < lexlen: # This code provides some short-circuit code for whitespace, tabs, and other ignored characters if lexdata[lexpos] in lexignore: + if lexdata[lexpos] == '\n': + self.lineno += 1 lexpos += 1 continue diff --git a/yacc.py b/yacc.py index 6699b7e..98377c7 100644 --- a/yacc.py +++ b/yacc.py @@ -8,7 +8,7 @@ # # Copyright (C) 2001, David M. Beazley # -# $Header: /cvsroot/ply/ply/yacc.py,v 1.15 2002/12/12 21:16:18 beazley Exp $ +# $Header: /home/esr/WWW/mad/RCS/yacc.py,v 1.1 2004/01/14 19:05:44 esr Exp esr $ # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -201,7 +201,7 @@ class Parser: lookahead = YaccSymbol() lookahead.type = '$' if debug: - print "%-20s : %s" % (lookahead, [xx.type for xx in symstack]) + sys.stderr.write("%-20s : %s\n" % (lookahead, [xx.type for xx in symstack])) # Check the action table s = statestack[-1] @@ -213,9 +213,11 @@ class Parser: # shift a symbol on the stack if ltype == '$': # Error, end of input - print "yacc: Parse error. EOF" + sys.stderr.write("yacc: Parse error. EOF\n") return statestack.append(t) + if debug > 1: + sys.stderr.write("yacc: shift state %s\n" % t) symstack.append(lookahead) lookahead = None @@ -235,6 +237,8 @@ class Parser: sym = YaccSymbol() sym.type = pname # Production name sym.value = None + if debug > 1: + sys.stderr.write("yacc: reduce %s\n" % sym.type) if plen: targ = symstack[-plen-1:] @@ -326,11 +330,11 @@ class Parser: if hasattr(errtoken,"lineno"): lineno = lookahead.lineno else: lineno = 0 if lineno: - print "yacc: Syntax error at line %d, token=%s" % (lineno, errtoken.type) + sys.stderr.write("yacc: Syntax error at line %d, token=%s\n" % (lineno, errtoken.type)) else: - print "yacc: Syntax error, token=%s" % errtoken.type + sys.stderr.write("yacc: Syntax error, token=%s" % errtoken.type) else: - print "yacc: Parse error in input. EOF" + sys.stderr.write("yacc: Parse error in input. EOF\n") return else: @@ -424,7 +428,7 @@ def validate_file(filename): if not prev: counthash[name] = linen else: - print "%s:%d: Function %s redefined. Previously defined on line %d" % (filename,linen,name,prev) + sys.stderr.write("%s:%d: Function %s redefined. Previously defined on line %d\n" % (filename,linen,name,prev)) noerror = 0 linen += 1 return noerror @@ -436,12 +440,12 @@ def validate_dict(d): if n[0:2] == 't_': continue if n[0:2] == 'p_': - print "yacc: Warning. '%s' not defined as a function" % n + sys.stderr.write("yacc: Warning. '%s' not defined as a function\n" % n) if isinstance(v,types.FunctionType) and v.func_code.co_argcount == 1: try: doc = v.__doc__.split(" ") if doc[1] == ':': - print "%s:%d: Warning. Possible grammar rule '%s' defined without p_ prefix." % (v.func_code.co_filename, v.func_code.co_firstlineno,n) + sys.stderr.write("%s:%d: Warning. Possible grammar rule '%s' defined without p_ prefix.\n" % (v.func_code.co_filename, v.func_code.co_firstlineno,n)) except StandardError: pass @@ -592,27 +596,27 @@ def is_identifier(s): def add_production(f,file,line,prodname,syms): if Terminals.has_key(prodname): - print "%s:%d: Illegal rule name '%s'. Already defined as a token." % (file,line,prodname) + sys.stderr.write("%s:%d: Illegal rule name '%s'. Already defined as a token.\n" % (file,line,prodname)) return -1 if prodname == 'error': - print "%s:%d: Illegal rule name '%s'. error is a reserved word." % (file,line,prodname) + sys.stderr.write("%s:%d: Illegal rule name '%s'. error is a reserved word.\n" % (file,line,prodname)) return -1 if not is_identifier(prodname): - print "%s:%d: Illegal rule name '%s'" % (file,line,prodname) + sys.stderr.write("%s:%d: Illegal rule name '%s'\n" % (file,line,prodname)) return -1 for s in syms: if not is_identifier(s) and s != '%prec': - print "%s:%d: Illegal name '%s' in rule '%s'" % (file,line,s, prodname) + sys.stderr.write("%s:%d: Illegal name '%s' in rule '%s'\n" % (file,line,s, prodname)) return -1 # See if the rule is already in the rulemap map = "%s -> %s" % (prodname,syms) if Prodmap.has_key(map): m = Prodmap[map] - print "%s:%d: Duplicate rule %s." % (file,line, m) - print "%s:%d: Previous definition at %s:%d" % (file,line, m.file, m.line) + sys.stderr.write("%s:%d: Duplicate rule %s.\n" % (file,line, m)) + sys.stderr.write("%s:%d: Previous definition at %s:%d\n" % (file,line, m.file, m.line)) return -1 p = Production() @@ -637,12 +641,12 @@ def add_production(f,file,line,prodname,syms): try: precname = p.prod[i+1] except IndexError: - print "%s:%d: Syntax error. Nothing follows %%prec." % (p.file,p.line) + sys.stderr.write("%s:%d: Syntax error. Nothing follows %%prec.\n" % (p.file,p.line)) return -1 prec = Precedence.get(precname,None) if not prec: - print "%s:%d: Nothing known about the precedence of '%s'" % (p.file,p.line,precname) + sys.stderr.write("%s:%d: Nothing known about the precedence of '%s'\n" % (p.file,p.line,precname)) return -1 else: p.prec = prec @@ -690,11 +694,11 @@ def add_function(f): error = 0 if f.func_code.co_argcount > 1: - print "%s:%d: Rule '%s' has too many arguments." % (file,line,f.__name__) + sys.stderr.write("%s:%d: Rule '%s' has too many arguments.\n" % (file,line,f.__name__)) return -1 if f.func_code.co_argcount < 1: - print "%s:%d: Rule '%s' requires an argument." % (file,line,f.__name__) + sys.stderr.write("%s:%d: Rule '%s' requires an argument.\n" % (file,line,f.__name__)) return -1 if f.__doc__: @@ -710,7 +714,7 @@ def add_function(f): if p[0] == '|': # This is a continuation of a previous rule if not lastp: - print "%s:%d: Misplaced '|'." % (file,dline) + sys.stderr.write("%s:%d: Misplaced '|'.\n" % (file,dline)) return -1 prodname = lastp if len(p) > 1: @@ -726,15 +730,15 @@ def add_function(f): else: syms = [ ] if assign != ':' and assign != '::=': - print "%s:%d: Syntax error. Expected ':'" % (file,dline) + sys.stderr.write("%s:%d: Syntax error. Expected ':'\n" % (file,dline)) return -1 e = add_production(f,file,dline,prodname,syms) error += e except StandardError: - print "%s:%d: Syntax error in rule '%s'" % (file,dline,ps) + sys.stderr.write("%s:%d: Syntax error in rule '%s'\n" % (file,dline,ps)) error -= 1 else: - print "%s:%d: No documentation string specified in function '%s'" % (file,line,f.__name__) + sys.stderr.write("%s:%d: No documentation string specified in function '%s'\n" % (file,line,f.__name__)) return error @@ -754,7 +758,7 @@ def compute_reachable(): for s in Nonterminals.keys(): if not Reachable[s]: - print "yacc: Symbol '%s' is unreachable." % s + sys.stderr.write("yacc: Symbol '%s' is unreachable.\n" % s) def mark_reachable_from(s, Reachable): ''' @@ -831,7 +835,7 @@ def compute_terminates(): # so it would be overkill to say that it's also non-terminating. pass else: - print "yacc: Infinite recursion detected for symbol '%s'." % s + sys.stderr.write("yacc: Infinite recursion detected for symbol '%s'.\n" % s) some_error = 1 return some_error @@ -848,7 +852,7 @@ def verify_productions(cycle_check=1): for s in p.prod: if not Prodnames.has_key(s) and not Terminals.has_key(s) and s != 'error': - print "%s:%d: Symbol '%s' used, but not defined as a token or a rule." % (p.file,p.line,s) + sys.stderr.write("%s:%d: Symbol '%s' used, but not defined as a token or a rule.\n" % (p.file,p.line,s)) error = 1 continue @@ -858,7 +862,7 @@ def verify_productions(cycle_check=1): _vf.write("Unused terminals:\n\n") for s,v in Terminals.items(): if s != 'error' and not v: - print "yacc: Warning. Token '%s' defined, but not used." % s + sys.stderr.write("yacc: Warning. Token '%s' defined, but not used.\n" % s) if yaccdebug: _vf.write(" %s\n"% s) unused_tok += 1 @@ -873,19 +877,19 @@ def verify_productions(cycle_check=1): for s,v in Nonterminals.items(): if not v: p = Prodnames[s][0] - print "%s:%d: Warning. Rule '%s' defined, but not used." % (p.file,p.line, s) + sys.stderr.write("%s:%d: Warning. Rule '%s' defined, but not used.\n" % (p.file,p.line, s)) unused_prod += 1 if unused_tok == 1: - print "yacc: Warning. There is 1 unused token." + sys.stderr.write("yacc: Warning. There is 1 unused token.\n") if unused_tok > 1: - print "yacc: Warning. There are %d unused tokens." % unused_tok + sys.stderr.write("yacc: Warning. There are %d unused tokens.\n" % unused_tok) if unused_prod == 1: - print "yacc: Warning. There is 1 unused rule." + sys.stderr.write("yacc: Warning. There is 1 unused rule.\n") if unused_prod > 1: - print "yacc: Warning. There are %d unused rules." % unused_prod + sys.stderr.write("yacc: Warning. There are %d unused rules.\n" % unused_prod) if yaccdebug: _vf.write("\nTerminals, with rules where they appear\n\n") @@ -954,16 +958,16 @@ def add_precedence(plist): prec = p[0] terms = p[1:] if prec != 'left' and prec != 'right' and prec != 'nonassoc': - print "yacc: Invalid precedence '%s'" % prec + sys.stderr.write("yacc: Invalid precedence '%s'\n" % prec) return -1 for t in terms: if Precedence.has_key(t): - print "yacc: Precedence already specified for terminal '%s'" % t + sys.stderr.write("yacc: Precedence already specified for terminal '%s'\n" % t) error += 1 continue Precedence[t] = (prec,plevel) except: - print "yacc: Invalid precedence table." + sys.stderr.write("yacc: Invalid precedence table.\n") error += 1 return error @@ -1242,7 +1246,7 @@ def slr_parse_table(): n_srconflict = 0 n_rrconflict = 0 - print "yacc: Generating SLR parsing table..." + sys.stderr.write("yacc: Generating SLR parsing table...\n") if yaccdebug: _vf.write("\n\nParsing method: SLR\n\n") @@ -1307,12 +1311,12 @@ def slr_parse_table(): if oldp.line > pp.line: action[st,a] = -p.number actionp[st,a] = p - # print "Reduce/reduce conflict in state %d" % st + # sys.stderr.write("Reduce/reduce conflict in state %d\n" % st) n_rrconflict += 1 _vfc.write("reduce/reduce conflict in state %d resolved using rule %d (%s).\n" % (st, actionp[st,a].number, actionp[st,a])) _vf.write(" ! reduce/reduce conflict for %s resolved using rule %d (%s).\n" % (a,actionp[st,a].number, actionp[st,a])) else: - print "Unknown conflict in state %d" % st + sys.stderr.write("Unknown conflict in state %d\n" % st) else: action[st,a] = -p.number actionp[st,a] = p @@ -1330,7 +1334,7 @@ def slr_parse_table(): # Whoa have a shift/reduce or shift/shift conflict if r > 0: if r != j: - print "Shift/shift conflict in state %d" % st + sys.stderr.write("Shift/shift conflict in state %d\n" % st) elif r < 0: # Do a precedence check. # - if precedence of reduce rule is higher, we reduce. @@ -1356,7 +1360,7 @@ def slr_parse_table(): _vf.write(" ! shift/reduce conflict for %s resolved as reduce.\n" % a) else: - print "Unknown conflict in state %d" % st + sys.stderr.write("Unknown conflict in state %d\n" % st) else: action[st,a] = j actionp[st,a] = p @@ -1395,13 +1399,13 @@ def slr_parse_table(): st += 1 if n_srconflict == 1: - print "yacc: %d shift/reduce conflict" % n_srconflict + sys.stderr.write("yacc: %d shift/reduce conflict\n" % n_srconflict) if n_srconflict > 1: - print "yacc: %d shift/reduce conflicts" % n_srconflict + sys.stderr.write("yacc: %d shift/reduce conflicts\n" % n_srconflict) if n_rrconflict == 1: - print "yacc: %d reduce/reduce conflict" % n_rrconflict + sys.stderr.write("yacc: %d reduce/reduce conflict\n" % n_rrconflict) if n_rrconflict > 1: - print "yacc: %d reduce/reduce conflicts" % n_rrconflict + sys.stderr.write("yacc: %d reduce/reduce conflicts\n" % n_rrconflict) # -----------------------------------------------------------------------------