From: Eric S. Raymond Date: Tue, 16 Sep 2003 17:27:44 +0000 (+0000) Subject: Report-parser logic now lives in a file of its own. X-Git-Tag: 2.0~38 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=17c33110c7b3cfaf422ebdeb5412a9dd02f7b636;p=comparator.git Report-parser logic now lives in a file of its own. --- diff --git a/filterator b/filterator index d9920f2..635ef9f 100755 --- a/filterator +++ b/filterator @@ -2,207 +2,7 @@ # # filterator -- filter and format the output of comparator # -import os, sys, re, exceptions - -# Tokens to be removed when checking whether a shred is significant -junk = { - "C": map(re.compile, ( - # Idioms that don't convey any meaning in isolation - r"return *\(?[a-z]+\)? *;", r"return *\(?-?[01]+\)? *;", - r"goto +[a-z]+;", r"exit *\([01]\);", - # Pragmas - r'/\* *ARGSUSED *\*/', r'/\* *NOTREACHED *\*/', - # Bare C keywords - r'\bbreak\b', r'\bcase\b',r'\bcontinue\b', r'\bdefault\b', - r'\bdo\b', r'\belse\b', r'\benum\b', r'\bif\b', r'\bgoto\b', - r'\breturn\b', r'\bswitch\b', r'\bwhile\b', - r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b', - r'\bstruct\b', r'typedef', r'\bunion\b', r'\bvoid\b', - r'# *define',r'# *endif',r'# *else',r'# *if\b', - r'# *ifdef\b',r'# *ifndef\b', - # Comment delimiters with no content - r'/\*+', r'\*+/', r'\*+', r'^ *\\s*\n', - # Common preprocessor macros, not significant by themselves. - r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b', - # Macro include lines are noise, too. - r'\s*#include.*',r'#\s*line.*', - # Common error macros. - r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b', - # Punctuation and whitespace - r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]', - r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+', - '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v' - )), - "shell": map(re.compile, ( - # Idioms that don't convey any meaning in isolation - r"exit *[01];?", - # Bare shell keywords - r"\bbreak\b", r"\bcase\b", r"\bdone\b", r"\bdo\b", r"\belse\b", - r"\besac\b", r"\bfi\b", r"\bif\b", r"\breturn\b", r"\bshift\b", - r"\btrue\b", r"\bwhile\b", - # Blank comment - "^#\n", - # Punctuation and whitespace - ':', ';', '\n','\t', '\v' - )), - } - -def nontrivial(filetype, text): - "Identify a shred as trivial or nontrivial." - if filetype: - if debug: - print "New shred, type %s:" % `filetype` - sys.stdout.write(text) - # Basic theory of this function is that if we throw out all C - # syntax and and common constants, and there is still an - # identifier, we're looking at something that might be - # interesting. - text = ' ' + text - while 1: - savecopy = text - for regexp in junk[filetype]: - text = regexp.sub(' ', text) - if savecopy == text: - break - else: - if debug: - print "Reduced:", `text` - continue - return text.strip() - -class ComparatorException(exceptions.Exception): - def __init__(self, message, retval=1): - self.message = message - self.retval = retval - -class CommonReport: - "Capture the state of common-segnent report." - # Template for parsing the output from comparator. - shredline = re.compile("(.*):([0-9]+):([0-9]+):([0-9]+)") - - def __init__(self, fp, dir): - # Read the SCF header - self.dir = dir - self.hash_method = "RXOR" - self.merge_program = None - id = fp.readline() - if not id.startswith("#SCF-B "): - raise ComparatorException("input is not a SCF-B file.\n") - while 1: - line = fp.readline() - if not line or line == '%%\n': - break - (tag, value) = line.split(":") - value = value.strip() - if tag == "Normalization": - self.normalization = value - elif tag == "Shred-Size": - self.shredsize = int(value) - elif tag == "Merge-Program": - self.merge_program = value - elif tag == "Hash-Method": - self.hash_method = value - # Read file statistics - self.trees = {} - while 1: - line = fp.readline() - if not line or line == '%%\n': - break - (tag, value) = line.split(":") - self.trees[tag] = int(value.strip()) - # Now read the common-segment stuff - self.cliques = [] - self.files = {} - locations = [] - while 1: - line = fp.readline() - if not line: - break - m = CommonReport.shredline.search(line) - if m: - (file, start, end, length) = \ - (m.group(1), int(m.group(2)), - int(m.group(3)), int(m.group(4))) - locations.append((file, start, end)) - self.files[file] = length - if line == '%%\n': - self.cliques.append(locations) - locations = [] - - def segment_count(self): - "Number of common chunks in the report." - return len(self.cliques) - - def line_count(self, tree): - "Number of common lines from a specified tree." - lines = 0 - for clique in self.cliques: - for (file, start, end) in clique: - # We find only the first match in the clique that - # is part of the specified tree. There could be others, - # and if whitespace removal is in effect they could - # have different names. - if file.startswith(tree): - lines += end - start + 1 - break - return lines - - def extract_text(self, clique): - "Return text corresponding to the given clique." - text = text_type = None - if self.dir: - olddir = os.getcwd() - os.chdir(dir) - for (file, start, end) in clique: - try: - # Try to deduce the text_type - text_type = None - if file.endswith(".c") or file.endswith(".h"): - text_type = "C" - # Skip to the relevant chunk - rfp = open(file) - for i in range(start-1): - line = rfp.readline() - if not text_type and i == 0: - if "sh" in line: - text_type = "shell" - text = "" - for i in range(start, end+1): - nextline = rfp.readline() - if not text_type and i == 0 and start == 1: - if "sh" in line: - text_type = "shell" - if nextline[0] == '%': - nextline = '%' + nextline - text += nextline - rfp.close() - break - except IOError: - pass - if self.dir: - os.chdir(olddir) - return (text_type, text) - - def filter_by_size(self, minsize): - "Throw out all common segments below a specified size." - filtered = [] - for clique in self.cliques: - for (file, start, end) in clique: - if end - start + 1 >= minsize: - filtered.append(clique) - break - self.cliques = filtered - - def filter_by_significance(self): - "Throw out all cliques that aren't significant." - filtered = [] - for clique in self.cliques: - (type, text) = self.extract_text(clique) - if text: - filtered.append(clique) - else: - raise ComparatorException("couldn't extracrt file text") - self.cliques = filtered +import sys, comparator if __name__ == '__main__': def report_common(state): @@ -257,7 +57,7 @@ if __name__ == '__main__': debug = 1 try: - state = CommonReport(sys.stdin, dir) + state = comparator.CommonReport(sys.stdin, dir) if minsize: state.filter_by_size(minsize) if dofilter: @@ -270,7 +70,7 @@ if __name__ == '__main__': print "%s:%d:%d" % (tree, lines, state.trees[tree]) else: report_common(state) - except ComparatorException, e: + except comparator.ComparatorException, e: sys.stderr.write("filterator: " + e.message + "\n") sys.exit(1)