Filtering witin class is fully in place.

diff --git a/filterator b/filterator
index dc464b7da32cae00499afe3a13f663101449aee6..50edb1b1c0f33baafbbde8a7ed7b56fa35770158 100755 (executable)
--- a/filterator
+++ b/filterator
@@ -4,6 +4,72 @@
 #
 import os, sys, re, getopt
 
+# Tokens to be removed when checking whether a shred is significant
+junk = {
+    "C": map(re.compile, (
+        # Idioms that don't convey any meaning in isolation
+        r"return *\(?[a-z]+\)? *;", r"return *\(?-?[01]+\)? *;",
+        r"goto +[a-z]+;", r"exit *\([01]\);",
+        # Pragmas
+        r'/\* *ARGSUSED *\*/', r'/\* *NOTREACHED *\*/', 
+        # Bare C keywords
+        r'\bbreak\b',  r'\bcase\b',r'\bcontinue\b', r'\bdefault\b',
+        r'\bdo\b', r'\belse\b', r'\benum\b', r'\bif\b', r'\bgoto\b',
+        r'\breturn\b', r'\bswitch\b', r'\bwhile\b',
+        r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b',
+        r'\bstruct\b', r'typedef', r'\bunion\b', r'\bvoid\b',
+        r'# *define',r'# *endif',r'# *else',r'# *if\b',
+        r'# *ifdef\b',r'# *ifndef\b',
+        # Comment delimiters with no content
+        r'/\*+', r'\*+/', r'\*+', r'^ *\\s*\n',
+        # Common preprocessor macros, not significant by themselves.
+        r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b',
+        # Macro include lines are noise, too.
+        r'\s*#include.*',r'#\s*line.*',
+        # Common error macros.
+        r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b',
+        # Punctuation and whitespace
+        r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]',
+        r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+', 
+        '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v'
+        )),
+    "shell": map(re.compile, (
+        # Idioms that don't convey any meaning in isolation
+        r"exit *[01];?",
+        # Bare shell keywords
+        r"\bbreak\b", r"\bcase\b", r"\bdone\b", r"\bdo\b", r"\belse\b",
+        r"\besac\b", r"\bfi\b", r"\bif\b", r"\breturn\b", r"\bshift\b",
+        r"\btrue\b", r"\bwhile\b", 
+        # Blank comment
+        "^#\n",
+        # Punctuation and whitespace
+        ':', ';', '\n','\t', '\v'
+        )),
+    }
+
+def nontrivial(filetype, text):
+    "Identify a shred as trivial or nontrivial."
+    if filetype:
+        if debug:
+            print "New shred, type %s:" % `filetype`
+            sys.stdout.write(text)
+        # Basic theory of this function is that if we throw out all C
+        # syntax and and common constants, and there is still an
+        # identifier, we're looking at something that might be
+        # interesting.
+        text = ' ' + text
+        while 1:
+            savecopy = text
+            for regexp in junk[filetype]:
+                text = regexp.sub(' ', text)
+            if savecopy == text:
+                break
+            else:
+                if debug:
+                    print "Reduced:", `text`
+                continue
+    return text.strip()
+
 class CommonReport:
     "Capture the state of common-segnent report."
     # Template for parsing the output from comparator.
@@ -121,75 +187,9 @@ class CommonReport:
     def filter_by_significance(self):
         "Throw out all cliques that aren't significant."
         self.cliques = filter(
-                       lambda x: apply(nontrival,self.extract_text(clique)),
+                       lambda x: apply(nontrivial,self.extract_text(x)),
                         self.cliques)
 
-# Tokens to be removed when checking whether a shred is significant
-junk = {
-    "C": map(re.compile, (
-        # Idioms that don't convey any meaning in isolation
-        r"return *\(?[a-z]+\)? *;", r"return *\(?-?[01]+\)? *;",
-        r"goto +[a-z]+;", r"exit *\([01]\);",
-        # Pragmas
-        r'/\* *ARGSUSED *\*/', r'/\* *NOTREACHED *\*/', 
-        # Bare C keywords
-        r'\bbreak\b',  r'\bcase\b',r'\bcontinue\b', r'\bdefault\b',
-        r'\bdo\b', r'\belse\b', r'\benum\b', r'\bif\b', r'\bgoto\b',
-        r'\breturn\b', r'\bswitch\b', r'\bwhile\b',
-        r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b',
-        r'\bstruct\b', r'typedef', r'\bunion\b', r'\bvoid\b',
-        r'# *define',r'# *endif',r'# *else',r'# *if\b',
-        r'# *ifdef\b',r'# *ifndef\b',
-        # Comment delimiters with no content
-        r'/\*+', r'\*+/', r'\*+', r'^ *\\s*\n',
-        # Common preprocessor macros, not significant by themselves.
-        r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b',
-        # Macro include lines are noise, too.
-        r'\s*#include.*',r'#\s*line.*',
-        # Common error macros.
-        r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b',
-        # Punctuation and whitespace
-        r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]',
-        r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+', 
-        '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v'
-        )),
-    "shell": map(re.compile, (
-        # Idioms that don't convey any meaning in isolation
-        r"exit *[01];?",
-        # Bare shell keywords
-        r"\bbreak\b", r"\bcase\b", r"\bdone\b", r"\bdo\b", r"\belse\b",
-        r"\besac\b", r"\bfi\b", r"\bif\b", r"\breturn\b", r"\bshift\b",
-        r"\btrue\b", r"\bwhile\b", 
-        # Blank comment
-        "^#\n",
-        # Punctuation and whitespace
-        ':', ';', '\n','\t', '\v'
-        )),
-    }
-
-def nontrivial(filetype, text):
-    "Identify a shred as trivial or nontrivial."
-    if filetype:
-        if debug:
-            print "New shred, type %s:" % `filetype`
-            sys.stdout.write(text)
-        # Basic theory of this function is that if we throw out all C
-        # syntax and and common constants, and there is still an
-        # identifier, we're looking at something that might be
-        # interesting.
-        text = ' ' + text
-        while 1:
-            savecopy = text
-            for regexp in junk[filetype]:
-                text = regexp.sub(' ', text)
-            if savecopy == text:
-                break
-            else:
-                if debug:
-                    print "Reduced:", `text`
-                continue
-    return text.strip()
-
 def report_common(state):
     "Generate a report on common code."
     print "Filter-Program: filterator 1.0"
@@ -224,7 +224,7 @@ if __name__ == '__main__':
         sys.stderr.write("usage: filterator [-d dir] [-m size] [-n] [-x]\n")
         sys.exit(2)
     stats = debug = 0
-    dofilter = 0
+    dofilter = 1
     minsize = 5
     linecounts = {}
     for (opt, val) in optlist: