From b7eb20987d7f63b6ea84832bd1dddbd95e3f3e14 Mon Sep 17 00:00:00 2001 From: "Eric S. Raymond" Date: Tue, 16 Sep 2003 16:55:08 +0000 Subject: [PATCH] Filtering witin class is fully in place. --- filterator | 136 ++++++++++++++++++++++++++--------------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/filterator b/filterator index dc464b7..50edb1b 100755 --- a/filterator +++ b/filterator @@ -4,6 +4,72 @@ # import os, sys, re, getopt +# Tokens to be removed when checking whether a shred is significant +junk = { + "C": map(re.compile, ( + # Idioms that don't convey any meaning in isolation + r"return *\(?[a-z]+\)? *;", r"return *\(?-?[01]+\)? *;", + r"goto +[a-z]+;", r"exit *\([01]\);", + # Pragmas + r'/\* *ARGSUSED *\*/', r'/\* *NOTREACHED *\*/', + # Bare C keywords + r'\bbreak\b', r'\bcase\b',r'\bcontinue\b', r'\bdefault\b', + r'\bdo\b', r'\belse\b', r'\benum\b', r'\bif\b', r'\bgoto\b', + r'\breturn\b', r'\bswitch\b', r'\bwhile\b', + r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b', + r'\bstruct\b', r'typedef', r'\bunion\b', r'\bvoid\b', + r'# *define',r'# *endif',r'# *else',r'# *if\b', + r'# *ifdef\b',r'# *ifndef\b', + # Comment delimiters with no content + r'/\*+', r'\*+/', r'\*+', r'^ *\\s*\n', + # Common preprocessor macros, not significant by themselves. + r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b', + # Macro include lines are noise, too. + r'\s*#include.*',r'#\s*line.*', + # Common error macros. + r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b', + # Punctuation and whitespace + r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]', + r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+', + '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v' + )), + "shell": map(re.compile, ( + # Idioms that don't convey any meaning in isolation + r"exit *[01];?", + # Bare shell keywords + r"\bbreak\b", r"\bcase\b", r"\bdone\b", r"\bdo\b", r"\belse\b", + r"\besac\b", r"\bfi\b", r"\bif\b", r"\breturn\b", r"\bshift\b", + r"\btrue\b", r"\bwhile\b", + # Blank comment + "^#\n", + # Punctuation and whitespace + ':', ';', '\n','\t', '\v' + )), + } + +def nontrivial(filetype, text): + "Identify a shred as trivial or nontrivial." + if filetype: + if debug: + print "New shred, type %s:" % `filetype` + sys.stdout.write(text) + # Basic theory of this function is that if we throw out all C + # syntax and and common constants, and there is still an + # identifier, we're looking at something that might be + # interesting. + text = ' ' + text + while 1: + savecopy = text + for regexp in junk[filetype]: + text = regexp.sub(' ', text) + if savecopy == text: + break + else: + if debug: + print "Reduced:", `text` + continue + return text.strip() + class CommonReport: "Capture the state of common-segnent report." # Template for parsing the output from comparator. @@ -121,75 +187,9 @@ class CommonReport: def filter_by_significance(self): "Throw out all cliques that aren't significant." self.cliques = filter( - lambda x: apply(nontrival,self.extract_text(clique)), + lambda x: apply(nontrivial,self.extract_text(x)), self.cliques) -# Tokens to be removed when checking whether a shred is significant -junk = { - "C": map(re.compile, ( - # Idioms that don't convey any meaning in isolation - r"return *\(?[a-z]+\)? *;", r"return *\(?-?[01]+\)? *;", - r"goto +[a-z]+;", r"exit *\([01]\);", - # Pragmas - r'/\* *ARGSUSED *\*/', r'/\* *NOTREACHED *\*/', - # Bare C keywords - r'\bbreak\b', r'\bcase\b',r'\bcontinue\b', r'\bdefault\b', - r'\bdo\b', r'\belse\b', r'\benum\b', r'\bif\b', r'\bgoto\b', - r'\breturn\b', r'\bswitch\b', r'\bwhile\b', - r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b', - r'\bstruct\b', r'typedef', r'\bunion\b', r'\bvoid\b', - r'# *define',r'# *endif',r'# *else',r'# *if\b', - r'# *ifdef\b',r'# *ifndef\b', - # Comment delimiters with no content - r'/\*+', r'\*+/', r'\*+', r'^ *\\s*\n', - # Common preprocessor macros, not significant by themselves. - r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b', - # Macro include lines are noise, too. - r'\s*#include.*',r'#\s*line.*', - # Common error macros. - r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b', - # Punctuation and whitespace - r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]', - r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+', - '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v' - )), - "shell": map(re.compile, ( - # Idioms that don't convey any meaning in isolation - r"exit *[01];?", - # Bare shell keywords - r"\bbreak\b", r"\bcase\b", r"\bdone\b", r"\bdo\b", r"\belse\b", - r"\besac\b", r"\bfi\b", r"\bif\b", r"\breturn\b", r"\bshift\b", - r"\btrue\b", r"\bwhile\b", - # Blank comment - "^#\n", - # Punctuation and whitespace - ':', ';', '\n','\t', '\v' - )), - } - -def nontrivial(filetype, text): - "Identify a shred as trivial or nontrivial." - if filetype: - if debug: - print "New shred, type %s:" % `filetype` - sys.stdout.write(text) - # Basic theory of this function is that if we throw out all C - # syntax and and common constants, and there is still an - # identifier, we're looking at something that might be - # interesting. - text = ' ' + text - while 1: - savecopy = text - for regexp in junk[filetype]: - text = regexp.sub(' ', text) - if savecopy == text: - break - else: - if debug: - print "Reduced:", `text` - continue - return text.strip() - def report_common(state): "Generate a report on common code." print "Filter-Program: filterator 1.0" @@ -224,7 +224,7 @@ if __name__ == '__main__': sys.stderr.write("usage: filterator [-d dir] [-m size] [-n] [-x]\n") sys.exit(2) stats = debug = 0 - dofilter = 0 + dofilter = 1 minsize = 5 linecounts = {} for (opt, val) in optlist: -- 2.43.0