Sent to the author.

diff --git a/lex.py b/lex.py
index 06df0b1da22c250146a35c91578d8b330b36ff38..e0b2519a89d316003074c853c85fed57f882e972 100644 (file)
--- a/lex.py
+++ b/lex.py
@@ -8,7 +8,7 @@
 #
 # Copyright (C) 2001, David M. Beazley
 #
-# $Header: /home/esr/src/ply-1.3.1/RCS/lex.py,v 1.1 2004/01/05 21:34:32 esr Exp $
+# $Header: /home/esr/WWW/mad/RCS/lex.py,v 1.1 2004/01/20 17:55:46 esr Exp esr $
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -154,7 +154,7 @@ if the input looks very corrupted.
 
 The lex module defines a t.lineno attribute on each token that can be used
 to track the current line number in the input.  The value of this
-variable is not modified by lex so it is up to your lexer module
+variable is not normally modified by lex so it is up to your lexer module
 to correctly update its value depending on the lexical properties
 of the input language.  To do this, you might write rules such as
 the following:
@@ -163,6 +163,9 @@ def t_newline(t):
     r' \n+ '
     t.lineno += t.value.count("\n")
 
+Exception: if you put \n in the ignore set, the lexer's internal line
+counter will be incremented each time \n is seen.
+
 To initialize your lexer so that it can be used, simply call the lex.lex()
 function in your rule file.  If there are any errors in your
 specification, warning messages or an exception will be generated to
@@ -287,6 +290,8 @@ class Lexer:
         while lexpos < lexlen:
             # This code provides some short-circuit code for whitespace, tabs, and other ignored characters
             if lexdata[lexpos] in lexignore:
+                if lexdata[lexpos] == '\n':
+                    self.lineno += 1
                 lexpos += 1
                 continue
 

diff --git a/yacc.py b/yacc.py
index 6699b7e013202627133d6be32478066061723fe2..98377c7b30a0901b688c70cd01a7499a26f0f147 100644 (file)
--- a/yacc.py
+++ b/yacc.py
@@ -8,7 +8,7 @@
 #
 # Copyright (C) 2001, David M. Beazley
 #
-# $Header: /cvsroot/ply/ply/yacc.py,v 1.15 2002/12/12 21:16:18 beazley Exp $
+# $Header: /home/esr/WWW/mad/RCS/yacc.py,v 1.1 2004/01/14 19:05:44 esr Exp esr $
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -201,7 +201,7 @@ class Parser:
                     lookahead = YaccSymbol()
                     lookahead.type = '$'
             if debug:
-                print "%-20s : %s" % (lookahead, [xx.type for xx in symstack])
+                sys.stderr.write("%-20s : %s\n" % (lookahead, [xx.type for xx in symstack]))
 
             # Check the action table
             s = statestack[-1]
@@ -213,9 +213,11 @@ class Parser:
                     # shift a symbol on the stack
                     if ltype == '$':
                         # Error, end of input
-                        print "yacc: Parse error. EOF"
+                        sys.stderr.write("yacc: Parse error. EOF\n")
                         return
                     statestack.append(t)
+                    if debug > 1:
+                        sys.stderr.write("yacc: shift state %s\n" % t)
                     symstack.append(lookahead)
                     lookahead = None
 
@@ -235,6 +237,8 @@ class Parser:
                     sym = YaccSymbol()
                     sym.type = pname       # Production name
                     sym.value = None
+                    if debug > 1:
+                        sys.stderr.write("yacc: reduce %s\n" % sym.type)
 
                     if plen:
                         targ = symstack[-plen-1:]
@@ -326,11 +330,11 @@ class Parser:
                             if hasattr(errtoken,"lineno"): lineno = lookahead.lineno
                             else: lineno = 0
                             if lineno:
-                                print "yacc: Syntax error at line %d, token=%s" % (lineno, errtoken.type)
+                                sys.stderr.write("yacc: Syntax error at line %d, token=%s\n" % (lineno, errtoken.type))
                             else:
-                                print "yacc: Syntax error, token=%s" % errtoken.type
+                                sys.stderr.write("yacc: Syntax error, token=%s" % errtoken.type)
                         else:
-                            print "yacc: Parse error in input. EOF"
+                            sys.stderr.write("yacc: Parse error in input. EOF\n")
                             return
 
                 else:
@@ -424,7 +428,7 @@ def validate_file(filename):
             if not prev:
                 counthash[name] = linen
             else:
-                print "%s:%d: Function %s redefined. Previously defined on line %d" % (filename,linen,name,prev)
+                sys.stderr.write("%s:%d: Function %s redefined. Previously defined on line %d\n" % (filename,linen,name,prev))
                 noerror = 0
         linen += 1
     return noerror
@@ -436,12 +440,12 @@ def validate_dict(d):
         if n[0:2] == 't_': continue
 
         if n[0:2] == 'p_':
-            print "yacc: Warning. '%s' not defined as a function" % n
+            sys.stderr.write("yacc: Warning. '%s' not defined as a function\n" % n)
         if isinstance(v,types.FunctionType) and v.func_code.co_argcount == 1:
             try:
                 doc = v.__doc__.split(" ")
                 if doc[1] == ':':
-                    print "%s:%d: Warning. Possible grammar rule '%s' defined without p_ prefix." % (v.func_code.co_filename, v.func_code.co_firstlineno,n)
+                    sys.stderr.write("%s:%d: Warning. Possible grammar rule '%s' defined without p_ prefix.\n" % (v.func_code.co_filename, v.func_code.co_firstlineno,n))
             except StandardError:
                 pass
 
@@ -592,27 +596,27 @@ def is_identifier(s):
 def add_production(f,file,line,prodname,syms):
     
     if Terminals.has_key(prodname):
-        print "%s:%d: Illegal rule name '%s'. Already defined as a token." % (file,line,prodname)
+        sys.stderr.write("%s:%d: Illegal rule name '%s'. Already defined as a token.\n" % (file,line,prodname))
         return -1
     if prodname == 'error':
-        print "%s:%d: Illegal rule name '%s'. error is a reserved word." % (file,line,prodname)
+        sys.stderr.write("%s:%d: Illegal rule name '%s'. error is a reserved word.\n" % (file,line,prodname))
         return -1
                 
     if not is_identifier(prodname):
-        print "%s:%d: Illegal rule name '%s'" % (file,line,prodname)
+        sys.stderr.write("%s:%d: Illegal rule name '%s'\n" % (file,line,prodname))
         return -1
 
     for s in syms:
         if not is_identifier(s) and s != '%prec':
-            print "%s:%d: Illegal name '%s' in rule '%s'" % (file,line,s, prodname)
+            sys.stderr.write("%s:%d: Illegal name '%s' in rule '%s'\n" % (file,line,s, prodname))
             return -1
 
     # See if the rule is already in the rulemap
     map = "%s -> %s" % (prodname,syms)
     if Prodmap.has_key(map):
         m = Prodmap[map]
-        print "%s:%d: Duplicate rule %s." % (file,line, m)
-        print "%s:%d: Previous definition at %s:%d" % (file,line, m.file, m.line)
+        sys.stderr.write("%s:%d: Duplicate rule %s.\n" % (file,line, m))
+        sys.stderr.write("%s:%d: Previous definition at %s:%d\n" % (file,line, m.file, m.line))
         return -1
 
     p = Production()
@@ -637,12 +641,12 @@ def add_production(f,file,line,prodname,syms):
             try:
                 precname = p.prod[i+1]
             except IndexError:
-                print "%s:%d: Syntax error. Nothing follows %%prec." % (p.file,p.line)
+                sys.stderr.write("%s:%d: Syntax error. Nothing follows %%prec.\n" % (p.file,p.line))
                 return -1
 
             prec = Precedence.get(precname,None)
             if not prec:
-                print "%s:%d: Nothing known about the precedence of '%s'" % (p.file,p.line,precname)
+                sys.stderr.write("%s:%d: Nothing known about the precedence of '%s'\n" % (p.file,p.line,precname))
                 return -1
             else:
                 p.prec = prec
@@ -690,11 +694,11 @@ def add_function(f):
     error = 0
     
     if f.func_code.co_argcount > 1:
-        print "%s:%d: Rule '%s' has too many arguments." % (file,line,f.__name__)
+        sys.stderr.write("%s:%d: Rule '%s' has too many arguments.\n" % (file,line,f.__name__))
         return -1
 
     if f.func_code.co_argcount < 1:
-        print "%s:%d: Rule '%s' requires an argument." % (file,line,f.__name__)
+        sys.stderr.write("%s:%d: Rule '%s' requires an argument.\n" % (file,line,f.__name__))
         return -1
           
     if f.__doc__:
@@ -710,7 +714,7 @@ def add_function(f):
                 if p[0] == '|':
                     # This is a continuation of a previous rule
                     if not lastp:
-                        print "%s:%d: Misplaced '|'." % (file,dline)
+                        sys.stderr.write("%s:%d: Misplaced '|'.\n" % (file,dline))
                         return -1
                     prodname = lastp
                     if len(p) > 1:
@@ -726,15 +730,15 @@ def add_function(f):
                     else:
                         syms = [ ]
                     if assign != ':' and assign != '::=':
-                        print "%s:%d: Syntax error. Expected ':'" % (file,dline)
+                        sys.stderr.write("%s:%d: Syntax error. Expected ':'\n" % (file,dline))
                         return -1
                 e = add_production(f,file,dline,prodname,syms)
                 error += e
             except StandardError:
-                print "%s:%d: Syntax error in rule '%s'" % (file,dline,ps)
+                sys.stderr.write("%s:%d: Syntax error in rule '%s'\n" % (file,dline,ps))
                 error -= 1
     else:
-        print "%s:%d: No documentation string specified in function '%s'" % (file,line,f.__name__)
+        sys.stderr.write("%s:%d: No documentation string specified in function '%s'\n" % (file,line,f.__name__))
     return error
 
 
@@ -754,7 +758,7 @@ def compute_reachable():
 
     for s in Nonterminals.keys():
         if not Reachable[s]:
-            print "yacc: Symbol '%s' is unreachable." % s
+            sys.stderr.write("yacc: Symbol '%s' is unreachable.\n" % s)
 
 def mark_reachable_from(s, Reachable):
     '''
@@ -831,7 +835,7 @@ def compute_terminates():
                 # so it would be overkill to say that it's also non-terminating.
                 pass
             else:
-                print "yacc: Infinite recursion detected for symbol '%s'." % s
+                sys.stderr.write("yacc: Infinite recursion detected for symbol '%s'.\n" % s)
                 some_error = 1
 
     return some_error
@@ -848,7 +852,7 @@ def verify_productions(cycle_check=1):
 
         for s in p.prod:
             if not Prodnames.has_key(s) and not Terminals.has_key(s) and s != 'error':
-                print "%s:%d: Symbol '%s' used, but not defined as a token or a rule." % (p.file,p.line,s)
+                sys.stderr.write("%s:%d: Symbol '%s' used, but not defined as a token or a rule.\n" % (p.file,p.line,s))
                 error = 1
                 continue
 
@@ -858,7 +862,7 @@ def verify_productions(cycle_check=1):
         _vf.write("Unused terminals:\n\n")
     for s,v in Terminals.items():
         if s != 'error' and not v:
-            print "yacc: Warning. Token '%s' defined, but not used." % s
+            sys.stderr.write("yacc: Warning. Token '%s' defined, but not used.\n" % s)
             if yaccdebug: _vf.write("   %s\n"% s)
             unused_tok += 1
 
@@ -873,19 +877,19 @@ def verify_productions(cycle_check=1):
     for s,v in Nonterminals.items():
         if not v:
             p = Prodnames[s][0]
-            print "%s:%d: Warning. Rule '%s' defined, but not used." % (p.file,p.line, s)
+            sys.stderr.write("%s:%d: Warning. Rule '%s' defined, but not used.\n" % (p.file,p.line, s))
             unused_prod += 1
 
     
     if unused_tok == 1:
-        print "yacc: Warning. There is 1 unused token."
+        sys.stderr.write("yacc: Warning. There is 1 unused token.\n")
     if unused_tok > 1:
-        print "yacc: Warning. There are %d unused tokens." % unused_tok
+        sys.stderr.write("yacc: Warning. There are %d unused tokens.\n" % unused_tok)
 
     if unused_prod == 1:
-        print "yacc: Warning. There is 1 unused rule."
+        sys.stderr.write("yacc: Warning. There is 1 unused rule.\n")
     if unused_prod > 1:
-        print "yacc: Warning. There are %d unused rules." % unused_prod
+        sys.stderr.write("yacc: Warning. There are %d unused rules.\n" % unused_prod)
 
     if yaccdebug:
         _vf.write("\nTerminals, with rules where they appear\n\n")
@@ -954,16 +958,16 @@ def add_precedence(plist):
             prec = p[0]
             terms = p[1:]
             if prec != 'left' and prec != 'right' and prec != 'nonassoc':
-                print "yacc: Invalid precedence '%s'" % prec
+                sys.stderr.write("yacc: Invalid precedence '%s'\n" % prec)
                 return -1
             for t in terms:
                 if Precedence.has_key(t):
-                    print "yacc: Precedence already specified for terminal '%s'" % t
+                    sys.stderr.write("yacc: Precedence already specified for terminal '%s'\n" % t)
                     error += 1
                     continue
                 Precedence[t] = (prec,plevel)
         except:
-            print "yacc: Invalid precedence table."
+            sys.stderr.write("yacc: Invalid precedence table.\n")
             error += 1
 
     return error
@@ -1242,7 +1246,7 @@ def slr_parse_table():
     n_srconflict = 0
     n_rrconflict = 0
 
-    print "yacc: Generating SLR parsing table..."
+    sys.stderr.write("yacc: Generating SLR parsing table...\n")
     if yaccdebug:
         _vf.write("\n\nParsing method: SLR\n\n")
         
@@ -1307,12 +1311,12 @@ def slr_parse_table():
                                     if oldp.line > pp.line:
                                         action[st,a] = -p.number
                                         actionp[st,a] = p
-                                    # print "Reduce/reduce conflict in state %d" % st
+                                    # sys.stderr.write("Reduce/reduce conflict in state %d\n" % st)
                                     n_rrconflict += 1
                                     _vfc.write("reduce/reduce conflict in state %d resolved using rule %d (%s).\n" % (st, actionp[st,a].number, actionp[st,a]))
                                     _vf.write("  ! reduce/reduce conflict for %s resolved using rule %d (%s).\n" % (a,actionp[st,a].number, actionp[st,a]))
                                 else:
-                                    print "Unknown conflict in state %d" % st
+                                    sys.stderr.write("Unknown conflict in state %d\n" % st)
                             else:
                                 action[st,a] = -p.number
                                 actionp[st,a] = p
@@ -1330,7 +1334,7 @@ def slr_parse_table():
                                 # Whoa have a shift/reduce or shift/shift conflict
                                 if r > 0:
                                     if r != j:
-                                        print "Shift/shift conflict in state %d" % st
+                                        sys.stderr.write("Shift/shift conflict in state %d\n" % st)
                                 elif r < 0:
                                     # Do a precedence check.
                                     #   -  if precedence of reduce rule is higher, we reduce.
@@ -1356,7 +1360,7 @@ def slr_parse_table():
                                             _vf.write("  ! shift/reduce conflict for %s resolved as reduce.\n" % a)
                                             
                                 else:
-                                    print "Unknown conflict in state %d" % st
+                                    sys.stderr.write("Unknown conflict in state %d\n" % st)
                             else:
                                 action[st,a] = j
                                 actionp[st,a] = p
@@ -1395,13 +1399,13 @@ def slr_parse_table():
         st += 1
 
     if n_srconflict == 1:
-        print "yacc: %d shift/reduce conflict" % n_srconflict
+        sys.stderr.write("yacc: %d shift/reduce conflict\n" % n_srconflict)
     if n_srconflict > 1:
-        print "yacc: %d shift/reduce conflicts" % n_srconflict        
+        sys.stderr.write("yacc: %d shift/reduce conflicts\n" % n_srconflict)
     if n_rrconflict == 1:
-        print "yacc: %d reduce/reduce conflict" % n_rrconflict
+        sys.stderr.write("yacc: %d reduce/reduce conflict\n" % n_rrconflict)
     if n_rrconflict > 1:
-        print "yacc: %d reduce/reduce conflicts" % n_rrconflict
+        sys.stderr.write("yacc: %d reduce/reduce conflicts\n" % n_rrconflict)
 
 
 # -----------------------------------------------------------------------------