From 7db5cde7bf984ceadab5daf84d49bec4f2f8e498 Mon Sep 17 00:00:00 2001 From: "Eric S. Raymond" Date: Tue, 27 Jan 2004 08:37:17 +0000 Subject: [PATCH] Sent to db. --- lex.py | 58 +++++++++++++++++++++++++++++---------------------------- yacc.py | 33 ++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/lex.py b/lex.py index e0b2519..b2d9f40 100644 --- a/lex.py +++ b/lex.py @@ -8,7 +8,7 @@ # # Copyright (C) 2001, David M. Beazley # -# $Header: /home/esr/WWW/mad/RCS/lex.py,v 1.1 2004/01/20 17:55:46 esr Exp esr $ +# $Header: /home/esr/WWW/mad/RCS/lex.py,v 1.2 2004/01/20 20:39:32 esr Exp esr $ # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -27,10 +27,11 @@ # See the file COPYING for a complete copy of the LGPL. # # -# This module automatically constructs a lexical analysis module from regular -# expression rules defined in a user-defined module. The idea is essentially the same -# as that used in John Aycock's Spark framework, but the implementation works -# at the module level rather than requiring the use of classes. +# This module automatically constructs a lexical analysis module from +# regular expression rules defined in a user-defined module. The idea +# is essentially the same as that used in John Aycock's Spark +# framework, but the implementation works at the module level rather +# than requiring the use of classes. # # This module tries to provide an interface that is closely modeled after # the traditional lex interface in Unix. It also differs from Spark @@ -125,8 +126,9 @@ For faster lexing, you can also define this in terms of the ignore set like this t_ignore = ' \t' -The characters in this string are ignored by the lexer. Use of this feature can speed -up parsing significantly since scanning will immediately proceed to the next token. +The characters in this string are ignored by the lexer. Use of this +feature can speed up parsing significantly since scanning will +immediately proceed to the next token. lex requires that the token returned by each rule has an attribute t.type. Other than this, rules are free to return any kind of token @@ -146,7 +148,7 @@ For instance, the following rule would print an error message and continue: def t_error(t): - print "Illegal character in input %s" % t.value[0] + sys.stderr.write("Illegal character in input %s\n" % t.value[0]) t.skip(1) Of course, a nice scanner might wish to skip more than one character @@ -186,7 +188,7 @@ this: Assuming that the module 'lexer' has initialized plex as shown above, parsing modules can safely import 'plex' without having -to import the rule file or any additional imformation about the +to import the rule file or any additional information about the scanner you have defined. """ @@ -206,9 +208,9 @@ class LexError(Exception): # Token class class LexToken: def __str__(self): - return "LexToken(%s,%r,%d)" % (self.type,self.value,self.lineno) + return "%s=%r" % (self.type,self.value) def __repr__(self): - return str(self) + return "LexToken(%s,%r,%d)" % (self.type,self.value,self.lineno) def skip(self,n): try: self._skipn += n @@ -387,7 +389,7 @@ def validate_file(filename): if not prev: counthash[name] = linen else: - print "%s:%d: Rule %s redefined. Previously defined on line %d" % (filename,linen,name,prev) + sys.stderr.write("%s:%d: Rule %s redefined. Previously defined on line %d." % (filename,linen,name,prev)) noerror = 0 linen += 1 return noerror @@ -473,17 +475,17 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): for n in tokens: if not is_identifier(n): - print "lex: Bad token name '%s'" % n + sys.stderr.write("lex: Bad token name '%s'\n" % n) error = 1 if lexer.lextokens.has_key(n): - print "lex: Warning. Token '%s' multiply defined." % n + sys.stderr.write("lex: Warning. Token '%s' multiply defined.\n" % n) lexer.lextokens[n] = None else: for n in tokens: lexer.lextokens[n] = None if debug: - print "lex: tokens = '%s'" % lexer.lextokens.keys() + sys.sdterr.write("lex: tokens = '%s'\n" % lexer.lextokens.keys()) # Get a list of symbols with the t_ prefix tsymbols = [f for f in ldict.keys() if f[:2] == 't_'] @@ -497,7 +499,7 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): elif isinstance(ldict[f],types.StringType): ssymbols.append((f,ldict[f])) else: - print "lex: %s not defined as a function or string" % f + sys.stderr.write("lex: %s not defined as a function or string\n" % f) error = 1 # Sort the functions by line number @@ -519,17 +521,17 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): if not optimize: if f.func_code.co_argcount > 1: - print "%s:%d: Rule '%s' has too many arguments." % (file,line,f.__name__) + sys.stderr.write("%s:%d: Rule '%s' has too many arguments.\n" % (file,line,f.__name__)) error = 1 continue if f.func_code.co_argcount < 1: - print "%s:%d: Rule '%s' requires an argument." % (file,line,f.__name__) + sys.stderr.write("%s:%d: Rule '%s' requires an argument.\n" % (file,line,f.__name__)) error = 1 continue if f.__name__ == 't_ignore': - print "%s:%d: Rule '%s' must be defined as a string." % (file,line,f.__name__) + sys.stderr.write("%s:%d: Rule '%s' must be defined as a string.\n" % (file,line,f.__name__)) error = 1 continue @@ -542,12 +544,12 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): try: c = re.compile(f.__doc__, re.VERBOSE) except re.error,e: - print "%s:%d: Invalid regular expression for rule '%s'. %s" % (file,line,f.__name__,e) + sys.stderr.write("%s:%d: Invalid regular expression for rule '%s'. %s\n" % (file,line,f.__name__,e)) error = 1 continue if debug: - print "lex: Adding rule %s -> '%s'" % (f.__name__,f.__doc__) + sys.stderr.write("lex: Adding rule %s -> '%s'\n" % (f.__name__,f.__doc__)) # Okay. The regular expression seemed okay. Let's append it to the master regular # expression we're building @@ -555,7 +557,7 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): if (regex): regex += "|" regex += "(?P<%s>%s)" % (f.__name__,f.__doc__) else: - print "%s:%d: No regular expression defined for rule '%s'" % (file,line,f.__name__) + sys.stderr.write("%s:%d: No regular expression defined for rule '%s'\n" % (file,line,f.__name__)) # Now add all of the simple rules for name,r in ssymbols: @@ -571,17 +573,17 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): continue if not lexer.lextokens.has_key(name[2:]): - print "lex: Rule '%s' defined for an unspecified token %s." % (name,name[2:]) + sys.stderr.write("lex: Rule '%s' defined for an unspecified token %s.\n" % (name,name[2:])) error = 1 continue try: c = re.compile(r,re.VERBOSE) except re.error,e: - print "lex: Invalid regular expression for rule '%s'. %s" % (name,e) + sys.stderr.write("lex: Invalid regular expression for rule '%s'. %s\n" % (name,e)) error = 1 continue if debug: - print "lex: Adding rule %s -> '%s'" % (name,r) + sys.stderr.write("lex: Adding rule %s -> '%s'\n" % (name,r)) if regex: regex += "|" regex += "(?P<%s>%s)" % (name,r) @@ -592,7 +594,7 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): error = 1 try: if debug: - print "lex: regex = '%s'" % regex + sys.stderr.write("lex: regex = '%s'\n" % regex) lexer.lexre = re.compile(regex, re.VERBOSE) # Build the index to function map for the matching engine @@ -634,12 +636,12 @@ def lex(module=None,debug=0,optimize=0,lextab="lextab"): lt.close() except re.error,e: - print "lex: Fatal error. Unable to compile regular expression rules. %s" % e + sys.stderr.write("lex: Fatal error. Unable to compile regular expression rules. %s\n" % e) error = 1 if error: raise SyntaxError,"lex: Unable to build lexer." if not lexer.lexerrorf: - print "lex: Warning. no t_error rule is defined." + sys.stderr.write("lex: Warning. no t_error rule is defined.\n") if not lexer.lexignore: lexer.lexignore = "" diff --git a/yacc.py b/yacc.py index 98377c7..b6299d6 100644 --- a/yacc.py +++ b/yacc.py @@ -8,7 +8,7 @@ # # Copyright (C) 2001, David M. Beazley # -# $Header: /home/esr/WWW/mad/RCS/yacc.py,v 1.1 2004/01/14 19:05:44 esr Exp esr $ +# $Header: /home/esr/WWW/mad/RCS/yacc.py,v 1.2 2004/01/20 20:39:32 esr Exp esr $ # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -201,7 +201,7 @@ class Parser: lookahead = YaccSymbol() lookahead.type = '$' if debug: - sys.stderr.write("%-20s : %s\n" % (lookahead, [xx.type for xx in symstack])) + errorlead = ("%s ^ %s" % (" ".join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip() # Check the action table s = statestack[-1] @@ -217,7 +217,7 @@ class Parser: return statestack.append(t) if debug > 1: - sys.stderr.write("yacc: shift state %s\n" % t) + sys.stderr.write("%-60s shift state %s\n" % (errorlead, t)) symstack.append(lookahead) lookahead = None @@ -238,7 +238,7 @@ class Parser: sym.type = pname # Production name sym.value = None if debug > 1: - sys.stderr.write("yacc: reduce %s\n" % sym.type) + sys.stderr.write("%-60s reduce %s\n" % (errorlead, sym.type)) if plen: targ = symstack[-plen-1:] @@ -295,17 +295,21 @@ class Parser: if t == 0: n = symstack[-1] return getattr(n,"value",None) + sys.stderr.write(errorlead, "\n") if t == None: - # We have some kind of parsing error here. To handle this, - # we are going to push the current token onto the tokenstack - # and replace it with an 'error' token. If there are any synchronization - # rules, they may catch it. + if debug: + sys.stderr.write(errorlead + "\n") + # We have some kind of parsing error here. To handle + # this, we are going to push the current token onto + # the tokenstack and replace it with an 'error' token. + # If there are any synchronization rules, they may + # catch it. # - # In addition to pushing the error token, we call call the user defined p_error() - # function if this is the first syntax error. This function is only called - # if errorcount == 0. - + # In addition to pushing the error token, we call call + # the user defined p_error() function if this is the + # first syntax error. This function is only called if + # errorcount == 0. if not self.errorcount: self.errorcount = error_count errtoken = lookahead @@ -320,8 +324,9 @@ class Parser: del errok, token, restart # Delete special functions if not self.errorcount: - # User must have done some kind of panic mode recovery on their own. The returned token - # is the next lookahead + # User must have done some kind of panic + # mode recovery on their own. The + # returned token is the next lookahead lookahead = tok errtoken = None continue -- 2.43.0