Report-parser logic now lives in a file of its own.

diff --git a/filterator b/filterator
index d9920f212a20736254dd6bc8ed21bf57ef8fe032..635ef9f9cd682423ad5926f87c1dfbc45edc6bf6 100755 (executable)
--- a/filterator
+++ b/filterator
@@ -2,207 +2,7 @@
 #
 # filterator -- filter and format the output of comparator
 #
-import os, sys, re, exceptions
-
-# Tokens to be removed when checking whether a shred is significant
-junk = {
-    "C": map(re.compile, (
-        # Idioms that don't convey any meaning in isolation
-        r"return *\(?[a-z]+\)? *;", r"return *\(?-?[01]+\)? *;",
-        r"goto +[a-z]+;", r"exit *\([01]\);",
-        # Pragmas
-        r'/\* *ARGSUSED *\*/', r'/\* *NOTREACHED *\*/', 
-        # Bare C keywords
-        r'\bbreak\b',  r'\bcase\b',r'\bcontinue\b', r'\bdefault\b',
-        r'\bdo\b', r'\belse\b', r'\benum\b', r'\bif\b', r'\bgoto\b',
-        r'\breturn\b', r'\bswitch\b', r'\bwhile\b',
-        r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b',
-        r'\bstruct\b', r'typedef', r'\bunion\b', r'\bvoid\b',
-        r'# *define',r'# *endif',r'# *else',r'# *if\b',
-        r'# *ifdef\b',r'# *ifndef\b',
-        # Comment delimiters with no content
-        r'/\*+', r'\*+/', r'\*+', r'^ *\\s*\n',
-        # Common preprocessor macros, not significant by themselves.
-        r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b',
-        # Macro include lines are noise, too.
-        r'\s*#include.*',r'#\s*line.*',
-        # Common error macros.
-        r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b',
-        # Punctuation and whitespace
-        r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]',
-        r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+', 
-        '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v'
-        )),
-    "shell": map(re.compile, (
-        # Idioms that don't convey any meaning in isolation
-        r"exit *[01];?",
-        # Bare shell keywords
-        r"\bbreak\b", r"\bcase\b", r"\bdone\b", r"\bdo\b", r"\belse\b",
-        r"\besac\b", r"\bfi\b", r"\bif\b", r"\breturn\b", r"\bshift\b",
-        r"\btrue\b", r"\bwhile\b", 
-        # Blank comment
-        "^#\n",
-        # Punctuation and whitespace
-        ':', ';', '\n','\t', '\v'
-        )),
-    }
-
-def nontrivial(filetype, text):
-    "Identify a shred as trivial or nontrivial."
-    if filetype:
-        if debug:
-            print "New shred, type %s:" % `filetype`
-            sys.stdout.write(text)
-        # Basic theory of this function is that if we throw out all C
-        # syntax and and common constants, and there is still an
-        # identifier, we're looking at something that might be
-        # interesting.
-        text = ' ' + text
-        while 1:
-            savecopy = text
-            for regexp in junk[filetype]:
-                text = regexp.sub(' ', text)
-            if savecopy == text:
-                break
-            else:
-                if debug:
-                    print "Reduced:", `text`
-                continue
-    return text.strip()
-
-class ComparatorException(exceptions.Exception):
-    def __init__(self, message, retval=1):
-        self.message = message
-        self.retval = retval
-
-class CommonReport:
-    "Capture the state of common-segnent report."
-    # Template for parsing the output from comparator.
-    shredline = re.compile("(.*):([0-9]+):([0-9]+):([0-9]+)")
-
-    def __init__(self, fp, dir):
-        # Read the SCF header
-        self.dir = dir
-        self.hash_method = "RXOR"
-        self.merge_program = None
-        id = fp.readline()
-        if not id.startswith("#SCF-B "):
-            raise ComparatorException("input is not a SCF-B file.\n")
-        while 1:
-            line = fp.readline()
-            if not line or line == '%%\n':
-                break
-            (tag, value) = line.split(":")
-            value = value.strip()
-            if tag == "Normalization":
-                self.normalization = value
-            elif tag == "Shred-Size":
-                self.shredsize = int(value)
-            elif tag == "Merge-Program":
-                self.merge_program = value
-            elif tag == "Hash-Method":
-                self.hash_method = value
-        # Read file statistics
-        self.trees = {}
-        while 1:
-            line = fp.readline()
-            if not line or line == '%%\n':
-                break
-            (tag, value) = line.split(":")
-            self.trees[tag] = int(value.strip())
-        # Now read the common-segment stuff
-        self.cliques = []
-        self.files = {}
-        locations = []
-        while 1:
-            line = fp.readline()
-            if not line:
-                break
-            m = CommonReport.shredline.search(line)
-            if m:
-                (file, start, end, length) = \
-                       (m.group(1), int(m.group(2)),
-                        int(m.group(3)), int(m.group(4)))
-                locations.append((file, start, end))
-                self.files[file] = length
-            if line == '%%\n':
-                self.cliques.append(locations)
-                locations = []
-
-    def segment_count(self):
-        "Number of common chunks in the report."
-        return len(self.cliques)
-
-    def line_count(self, tree):
-        "Number of common lines from a specified tree."
-        lines = 0
-        for clique in self.cliques:
-            for (file, start, end) in clique:
-                # We find only the first match in the clique that
-                # is part of the specified tree.  There could be others,
-                # and if whitespace removal is in effect they could
-                # have different names.
-                if file.startswith(tree):
-                    lines += end - start + 1
-                    break
-        return lines
-
-    def extract_text(self, clique):
-        "Return text corresponding to the given clique."
-        text = text_type = None
-        if self.dir:
-            olddir = os.getcwd()
-            os.chdir(dir)
-        for (file, start, end) in clique:
-            try:
-                # Try to deduce the text_type
-                text_type = None
-                if file.endswith(".c") or file.endswith(".h"):
-                    text_type = "C"
-                # Skip to the relevant chunk
-                rfp = open(file)
-                for i in range(start-1):
-                    line = rfp.readline()
-                    if not text_type and i == 0:
-                        if "sh" in line:
-                            text_type = "shell"
-                text = ""
-                for i in range(start, end+1):
-                    nextline = rfp.readline()
-                    if not text_type and i == 0 and start == 1:
-                        if "sh" in line:
-                            text_type = "shell"
-                    if nextline[0] == '%':
-                        nextline = '%' + nextline
-                    text += nextline
-                rfp.close()
-                break
-            except IOError:
-                pass
-        if self.dir:
-            os.chdir(olddir)
-        return (text_type, text)
-
-    def filter_by_size(self, minsize):
-        "Throw out all common segments below a specified size."
-        filtered = []
-        for clique in self.cliques:
-            for (file, start, end) in clique:
-                if end - start + 1 >= minsize:
-                    filtered.append(clique)
-                    break
-        self.cliques = filtered
-
-    def filter_by_significance(self):
-        "Throw out all cliques that aren't significant."
-        filtered = []
-        for clique in self.cliques:
-            (type, text) = self.extract_text(clique)
-            if text:
-                filtered.append(clique)
-            else:
-                raise ComparatorException("couldn't extracrt file text")
-        self.cliques = filtered
+import sys, comparator
 
 if __name__ == '__main__':
     def report_common(state):
@@ -257,7 +57,7 @@ if __name__ == '__main__':
             debug = 1
 
     try:
-        state = CommonReport(sys.stdin, dir)
+        state = comparator.CommonReport(sys.stdin, dir)
         if minsize:
             state.filter_by_size(minsize)
         if dofilter:
@@ -270,7 +70,7 @@ if __name__ == '__main__':
                 print "%s:%d:%d" % (tree, lines, state.trees[tree])
         else:
             report_common(state)
-    except ComparatorException, e:
+    except comparator.ComparatorException, e:
         sys.stderr.write("filterator: " + e.message + "\n")
         sys.exit(1)