From: Eric S. Raymond Date: Wed, 10 Mar 2004 04:59:18 +0000 (+0000) Subject: Added -f and -d options. X-Git-Tag: 2.4~15 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=b9db48b5d9f8f9eefbf33e914ffcaa6debf06e5f;p=comparator.git Added -f and -d options. --- diff --git a/comparator.py b/comparator.py index 77f6b70..919807b 100644 --- a/comparator.py +++ b/comparator.py @@ -23,6 +23,7 @@ class CommonReport: self.dir = dir self.hash_method = "RXOR" self.merge_program = None + self.filtering = None id = self.fp.readline() if not id.startswith("#SCF-B "): raise ComparatorException("input is not a SCF-B file.\n") @@ -42,6 +43,8 @@ class CommonReport: self.hash_method = value elif tag == "Matches": self.matches = int(value) + elif tag == "Filtering": + self.filtering = value # Read file properties self.trees = {} while 1: @@ -108,13 +111,12 @@ class CommonReport: os.chdir(olddir) return (file, start, end, text) - def filter_by_size(self, minsize): - "Throw out all common segments below a specified size." + def cliquefilter(self, predicate): self.read_matches() filtered = [] for clique in self.cliques: for (file, start, end) in clique: - if end - start + 1 >= minsize: + if predicate(file, start, end): filtered.append(clique) break else: @@ -123,6 +125,44 @@ class CommonReport: properties['matchlines'] -= end - start + 1 self.cliques = filtered + def filter_by_size(self, minsize): + "Throw out all common segments below a specified size." + self.cliquefilter(lambda file, start, end: end - start + 1 >= minsize) + + def filter_by_filename(self, regexp): + "Filter by name of involved file." + self.cliquefilter(lambda file, start, end: regexp.search(file)) + + def metadump(self, fp=sys.stdout, divider="%%n", fmt=None): + "Dump the header in a canonical format." + fp.write("Filter-Program: filterator 1.0\n") + fp.write("Hash-Method: RXOR\n") + if self.merge_program: + fp.write("Merge-Program: %s\n" % self.merge_program) + fp.write("Normalization: %s\n" % self.normalization) + fp.write("Shred-Size: %d\n" % self.shredsize) + fp.write("Filtering: %s\n" % self.filtering) + #if minsize: + # fp.write("Minimum-Size: %d\n" % minsize) + fp.write(divider) + for tree in self.trees: + rep = tree + ":" + for (key, value) in self.trees[tree].items(): + rep += " %s=%s," % (key, value) + fp.write(rep[:-1] + "\n") + fp.write(divider) + if fmt: + for clique in self.cliques: + for (file, start, end) in clique: + fp.write(fmt(file, start, end, self.files[file])) + fp.write(divider) + + def dump(self, fp): + "Dump this in the same format as the input." + fp.write("#SCF-B\n") + self.metadump(fp=fp, divider="%%\n", + fmt=lambda f,s,e, z: "%s:%d:%d:%d\n" % (f,s,e,z)) + # Property flags C_CODE = 0x01 SHELL_CODE = 0x02 diff --git a/comparator.spec b/comparator.spec index 12d2668..23bcbbf 100644 --- a/comparator.spec +++ b/comparator.spec @@ -1,6 +1,6 @@ Name: comparator Summary: fast comparison of large source-code trees -Version: 2.3 +Version: 2.4 Release: 1 License: GPL Group: Utilities @@ -24,7 +24,9 @@ make %{?_smp_mflags} comparator comparator.1 [ "$RPM_BUILD_ROOT" -a "$RPM_BUILD_ROOT" != / ] && rm -rf "$RPM_BUILD_ROOT" mkdir -p "$RPM_BUILD_ROOT"%{_bindir} mkdir -p "$RPM_BUILD_ROOT"%{_mandir}/man1/ -pylib=`ls -d %{_libdir}/python*/site-packages/` +set -- `ls -d %{_libdir}/python*/site-packages/` +while [ "$2" ]; do shift; done; +pylib=$1 mkdir -p "$RPM_BUILD_ROOT"${pylib} cp comparator filterator "$RPM_BUILD_ROOT"%{_bindir} cp comparator.1 "$RPM_BUILD_ROOT"%{_mandir}/man1/ @@ -42,6 +44,10 @@ cp comparator.py "$RPM_BUILD_ROOT"${pylib} %{_libdir}/python*/site-packages/comparator.py %changelog +* Tue Mar 9 2004 Eric S. Raymond 2.4-1 +- Fixed strtok() bug apparently introduced by C library change. + Added -d and -f options to filterator. + * Mon Dec 29 2003 Eric S. Raymond 2.3-1 - Source RPMs no longer depend on myversion symbol. diff --git a/comparator.xml b/comparator.xml index 835c0c0..becf0f5 100644 --- a/comparator.xml +++ b/comparator.xml @@ -197,15 +197,28 @@ of shell keywords and punctuation. filterator is a postprocessor for the SCF-B output of comparator that produces an -actual code listing. The acts as it does for -comparator. The option -sets the minimum size of overlap to be reported. Setting this to a -slightly larger number than the comparator -shred size is a useful way to filter out spurious matches; thus, while +actual code listing. + +The acts as it does for +comparator. + +The option sets the minimum size of overlap +to be reported. Setting this to a slightly larger number than the +comparator shred size is a useful way to +filter out spurious matches; thus, while comparator defaults to a shred size of 3, filterator defaults to a minimum size of 5. +The -f option takes a Python regexp as argument and throws out +all cliques for which no assocoated filename matches the regexp. This +might be useful if you want to see a common-segment report only for +files with a particular extension such as .h or .S. + +The -n option suppresses code listing, so that an SCF-B output +is produced instead. You can use this to filter common-segment +reports with -m and -f options of the tool. + Each filterator line beginning with % is the filename from which the following common span of lines is displayed filterator displays each span of lines, diff --git a/filterator b/filterator index 94c6ccb..6540676 100755 --- a/filterator +++ b/filterator @@ -2,58 +2,70 @@ # # filterator -- filter and format the output of comparator # -import sys, comparator +import sys, comparator, re + +def report_common(state): + "Generate a report on common code." + print "Filter-Program: filterator 1.0" + print "Hash-Method: RXOR" + if state.merge_program: + print "Merge-Program:", state.merge_program + print "Normalization:", state.normalization + print "Shred-Size: %d" % state.shredsize + if minsize: + print "Minimum-Size: %d" % minsize + print 75 * '-' + for tree in state.trees: + rep = tree + ":" + for (key, value) in state.trees[tree].items(): + rep += " %s=%s," % (key, value) + sys.stdout.write(rep[:-1] + "\n") + print 75 * '-' + state.read_matches() + for clique in state.cliques: + (file, start, end, text) = state.extract_text(clique) + if start == 1 and end == state.files[file]: + print "%% %s:%d-%d: entire (%d matches)" % \ + (file, start, end, len(clique)) + else: + print "%% %s:%d-%d: (%d matches)" % \ + (file, start, end, len(clique)) + sys.stdout.write(text) if __name__ == '__main__': - def report_common(state): - "Generate a report on common code." - print "Filter-Program: filterator 1.0" - print "Hash-Method: MD5" - if state.merge_program: - print "Merge-Program:", state.merge_program - print "Normalization:", state.normalization - print "Shred-Size: %d" % state.shredsize - if minsize: - print "Minimum-Size: %d" % minsize - print 75 * '-' - for tree in state.trees: - rep = tree + ":" - for (key, value) in state.trees[tree].items(): - rep += " %s=%s," % (key, value) - sys.stdout.write(rep[:-1] + "\n") - print 75 * '-' - state.read_matches() - for clique in state.cliques: - (file, start, end, text) = state.extract_text(clique) - if start == 1 and end == state.files[file]: - print "%% %s:%d-%d: entire (%d matches)" % \ - (file, start, end, len(clique)) - else: - print "%% %s:%d-%d: (%d matches)" % \ - (file, start, end, len(clique)) - sys.stdout.write(text) # Main line begins here import getopt try: - (optlist, args) = getopt.getopt(sys.argv[1:], 'd:m:') + (optlist, args) = getopt.getopt(sys.argv[1:], 'd:f:m:n') except getopt.GetoptError: - sys.stderr.write("usage: filterator [-d dir] [-m size]\n") + sys.stderr.write("usage: filterator [-n] [-d dir] [-m size] [-f filter]\n") sys.exit(2) - minsize = 5 dir = None + filter = None + minsize = 5 + expand = True linecounts = {} for (opt, val) in optlist: if opt == '-d': dir = val + elif opt == '-f': + filter = val elif opt == '-m': minsize = int(val) + elif opt == '-n': + expand = False try: state = comparator.CommonReport(sys.stdin, dir) if minsize: state.filter_by_size(minsize) + if filter: + state.filter_by_filename(re.compile(filter)) try: - report_common(state) + if expand: + report_common(state) + else: + state.dump(sys.stdout) except (KeyboardInterrupt, IOError): pass except comparator.ComparatorException, e: