VERS=2.0
-CODE = shredtree.c shred.h report.c hash.c main.c md5.c md5.h \
+CODE = shredtree.c shred.h report.c hash.c filter.c main.c md5.c md5.h \
hash.h hashtab.h \
filterator
SCRIPTS = hashgen.py setup.py
EXTRAS = shredtree.py shredcompare.py
TEST = test
SOURCES = $(CODE) $(DOCS) $(EXTRAS) $(TEST) comparator.spec Makefile
-CFLAGS = -O3
+CFLAGS = -g #-O3
LDFLAGS =
all: comparator comparator.1
main.o: main.c shred.h hash.h
$(CC) -DVERSION=\"$(VERS)\" -c $(CFLAGS) main.c
+filter.o: filter.c
+ $(CC) -c $(CFLAGS) filter.c
hash.o: hash.c hash.h hashtab.h
$(CC) -c $(CFLAGS) hash.c
shredtree.o: shredtree.c shred.h hash.h
$(CC) -c $(CFLAGS) shredtree.c
report.o: report.c shred.h hash.h
$(CC) -c $(CFLAGS) report.c
-comparator: main.o hash.o shredtree.o md5.o report.o
- $(CC) $(CFLAGS) main.o hash.o shredtree.o md5.o report.o $(LDFLAGS) -o comparator
+comparator: main.o hash.o filter.o shredtree.o md5.o report.o
+ $(CC) $(CFLAGS) main.o hash.o filter.o shredtree.o md5.o report.o $(LDFLAGS) -o comparator
hashtab.h: hashgen.py
python hashgen.py >hashtab.h
+filter: filter.c
+ $(CC) -DTEST $(CFLAGS) -o filter filter.c
+
clean:
- rm -f comparator shredtree.o md5.o report.o main.o hash.o *~
+ rm -f comparator filter *.o *~
rm -f comparator.1 filterator.1
rm -f *.dump *.scf
import os, sys, re, exceptions
-# Tokens to be removed when checking whether a shred is significant
-junk = {
- "C": map(re.compile, (
- # Idioms that don't convey any meaning in isolation
- r"return *\(?[a-z]+\)? *;", r"return *\(?-?[01]+\)? *;",
- r"goto +[a-z]+;", r"exit *\([01]\);",
- # Pragmas
- r'/\* *ARGSUSED *\*/',
- r'/\* *NOTREACHED *\*/',
- r'/\* *FALL *THRO?UG?H? *\*/',
- # Bare C keywords
- r'\bbreak\b', r'\bcase\b',r'\bcontinue\b', r'\bdefault\b',
- r'\bdo\b', r'\belse\b', r'\benum\b', r'\bif\b', r'\bgoto\b',
- r'\breturn\b', r'\bswitch\b', r'\bwhile\b',
- r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b',
- r'\bstruct\b', r'\btypedef\b', r'\bunion\b', r'\bvoid\b',
- r'# *define',r'# *endif',r'# *else',r'# *if\b',
- r'# *ifdef\b',r'# *ifndef\b',
- # Comment delimiters with no content
- r'/\*+', r'\*+/', r'\*+', r'^ *\\s*\n',
- # Common preprocessor macros, not significant by themselves.
- r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b',
- # Macro include lines are noise, too.
- r'\s*#include.*',r'#\s*line.*',
- # Common error macros.
- r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b',
- # Punctuation and whitespace
- r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]',
- r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+',
- '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v'
- )),
- "shell": map(re.compile, (
- # Idioms that don't convey any meaning in isolation
- r"exit *[01];?",
- # Bare shell keywords
- r"\bbreak\b", r"\bcase\b", r"\bdone\b", r"\bdo\b", r"\belse\b",
- r"\besac\b", r"\bfi\b", r"\bif\b", r"\breturn\b", r"\bshift\b",
- r"\btrue\b", r"\bwhile\b",
- # Blank comment
- "^#\n",
- # Punctuation and whitespace
- ':', ';', '\n','\t', '\v'
- )),
- }
-
-def nontrivial(filetype, text, debug=None):
- "Identify a shred as trivial or nontrivial."
- if filetype:
- if debug:
- print "New shred, type %s:" % `filetype`
- sys.stdout.write(text)
- # Basic theory of this function is that if we throw out all C
- # syntax and and common constants, and there is still an
- # identifier, we're looking at something that might be
- # interesting.
- text = ' ' + text
- while 1:
- savecopy = text
- for regexp in junk[filetype]:
- text = regexp.sub(' ', text)
- if savecopy == text:
- break
- else:
- if debug:
- print "Reduced:", `text`
- continue
- return text.strip()
-
class ComparatorException(exceptions.Exception):
def __init__(self, message, retval=1):
self.message = message
self.merge_program = value
elif tag == "Hash-Method":
self.hash_method = value
+ elif tag == "Matches":
+ self.matches = int(value)
# Read file properties
self.trees = {}
while 1:
if self.name:
self.fp.close()
- def segment_count(self):
- "Number of common chunks in the report."
- return len(self.cliques)
-
- def line_count(self, tree):
- "Number of common lines from a specified tree."
- lines = 0
- for clique in self.cliques:
- for (file, start, end) in clique:
- # We find only the first match in the clique that
- # is part of the specified tree. There could be others,
- # and if whitespace removal is in effect they could
- # have different names.
- if file.startswith(tree):
- lines += end - start + 1
- break
- return lines
-
def extract_text(self, clique):
"Return text corresponding to the given clique."
- text = text_type = None
+ text = None
if self.dir:
olddir = os.getcwd()
os.chdir(self.dir)
for (file, start, end) in clique:
try:
- # Try to deduce the text_type
- text_type = None
- if file.endswith(".c") or file.endswith(".h"):
- text_type = "C"
# Skip to the relevant chunk
rfp = open(file)
for i in range(start-1):
line = rfp.readline()
- if not text_type and i == 0:
- if "sh" in line:
- text_type = "shell"
text = ""
for i in range(start, end+1):
nextline = rfp.readline()
- if not text_type and i == 0 and start == 1:
- if "sh" in line:
- text_type = "shell"
if nextline[0] == '%':
nextline = '%' + nextline
text += nextline
os.chdir(olddir)
if not text:
print >>sys.stderr, "Failed to extract text from", clique
- return (text_type, text)
+ return text
def filter_by_size(self, minsize):
"Throw out all common segments below a specified size."
break
self.cliques = filtered
- def filter_by_significance(self):
- "Throw out all cliques that aren't significant."
- filtered = []
- for clique in self.cliques:
- (type, text) = self.extract_text(clique)
- if nontrivial(type, text):
- filtered.append(clique)
- self.cliques = filtered
-
- def statistics(self):
- "Return a statistical summary object for the current report."
- return CommonStatistics(self)
-
-class CommonStatistics:
- "Statistical information from a report."
- def __init__(self, report):
- self.chunks = report.segment_count()
- self.trees = []
- self.normalization = report.normalization
- for (tree, chunks, lines, totallines) in report.trees.items():
- lines = report.line_count(tree)
- self.trees.append((tree, lines, totallines))
+# Property flags
+C_CODE = 0x01
+SHELL_CODE = 0x02
+SIGNIFICANT = 0x40
- def __str__(self):
- rep = "Trees: " + " ".join(map(lambda x: x[0], self.trees))
- rep = "%d overlaps with %s normalization\n" % (self.chunks, self.normalization)
- for tree in self.trees:
- lines = tree['lines']
- total = tree['total']
- rep += "%s:\t%d of %d (%02.2f%%)\n" % (tree, lines, total, (lines * 100.0)/total)
- return rep
+# End
<arg choice='opt'>-C</arg>
<arg choice='opt'>-d <replaceable>dir</replaceable></arg>
<arg choice='opt'>-m <replaceable>size</replaceable></arg>
+ <arg choice='opt'>-n</arg>
<arg choice='opt'>-o <replaceable>file</replaceable></arg>
<arg choice='opt'>-r</arg>
<arg choice='opt'>-s <replaceable>shredsize</replaceable></arg>
- <arg choice='opt'>-w</arg>
<arg choice='opt'>-v</arg>
+ <arg choice='opt'>-w</arg>
<arg choice='opt'>-x</arg>
<arg choice='plain' rep='repeat'>path</arg>
<sbr/>
(including blank lines) to be ignored for comparison purposes (line
numbers in the output report will nevertheless be correct).</para>
-<para>The <option>-C</option> implies <option>-w</option> and also
+<para>Normally, <application>comparator</application> performs
+insignificance filtering before emitting a span into the common-segment
+report. The <option>-n</option> suppresses this, emitting all common
+spans. See the discussion of significance filtering below.</para>
+
+<para>The <option>-C</option> option implies <option>-w</option> and also
applies C-language-specific normalization; { and } characters are
removed. This is recommended for comparing C code; it prevents the
comparison from being fooled by differences in indent style.</para>
90%.</para>
</refsect2>
+<refsect2><title>Significance filtering</title>
+
+<para>Normally, <application>comparator</application> tries to
+throw out common segments that are just noise. A segment is
+considered noise if one of the following conditions holds:</para>
+
+<orderedlist>
+<listitem><para>Its file has a .c or .h extension, and the segment
+consists entirely of whitespace, punctuation, C keywords, C include
+lines, and other things that are not C identifiers, function names, or
+user-defined macros.</para></listitem>
+
+<listitem><para>The file has a .sh extension or a #! first line
+directing shell interpretation, and the segment consists entirely
+of shell keywords and punctuation.</para></listitem>
+</orderedlist>
+
+</refsect2>
+
<refsect2><title>filterator</title>
<para><application>filterator</application> is a postprocessor for the
#include lines and a few common macros and see if anything is left.
The noise test for shell files throws away Bourne-shell keywords</para>
-<para>The <option>-n</option> suppresses noise filtering. The
+<para>The
<option>-d</option> acts as it does for
<application>comparator</application>. The <option>-m</option> option
sets the minimum size of overlap to be reported. Setting this to a
shred size is a useful way to filter out spurious matches; thus, while
<application>comparator</application> defaults to a shred size of 3,
<application>filterator</application> defaults to a minimum size of
-5. The <option>-x</option> enables some debugging output. It is for
-developers; if you need to understand it, read the code.</para>
+5.</para>
<para>Each <application>filterator</application> line beginning with %
is the filename from which the following common span of lines is displayed
def report_common(state):
"Generate a report on common code."
print "Filter-Program: filterator 1.0"
- print "Filtering:", ("none", "language")[dofilter]
print "Hash-Method: MD5"
if state.merge_program:
print "Merge-Program:", state.merge_program
print "Shred-Size: %d" % state.shredsize
if minsize:
print "Minimum-Size: %d" % minsize
- print 75 * '-'
- sys.stdout.write(str(comparator.CommonStatistics(state)))
+ #print 75 * '-'
+ #sys.stdout.write(str(comparator.CommonStatistics(state)))
print 75 * '-'
for clique in state.cliques:
for (file, start, end) in clique:
print "%% %s:%d-%d: entire (%d matches)" % \
(file, start, end, len(clique))
continue
- (text_type, text) = state.extract_text(clique)
print "%% %s:%d-%d: (%d matches)" % (file, start, end, len(clique))
- sys.stdout.write(state.extract_text(clique)[1])
+ sys.stdout.write(state.extract_text(clique))
# Main line begins here
import getopt
try:
- (optlist, args) = getopt.getopt(sys.argv[1:], 'd:ns:Sx')
+ (optlist, args) = getopt.getopt(sys.argv[1:], 'd:m')
except getopt.GetoptError:
- sys.stderr.write("usage: filterator [-d dir] [-m size] [-n] [-x]\n")
+ sys.stderr.write("usage: filterator [-d dir] [-m size]\n")
sys.exit(2)
- stats = debug = 0
- dofilter = 1
minsize = 5
dir = None
linecounts = {}
dir = val
elif opt == '-m':
minsize = int(val)
- elif opt == '-n':
- dofilter = 0
- elif opt == '-S':
- stats = 1
- elif opt == '-x':
- debug = 1
try:
state = comparator.CommonReport(sys.stdin, dir)
- if minsize:
- state.filter_by_size(minsize)
- if dofilter:
- state.filter_by_significance()
-
- if stats:
- sys.stdout.write(str(state.statistics()))
- else:
- report_common(state)
+ #if minsize:
+ # state.filter_by_size(minsize)
+ report_common(state)
except comparator.ComparatorException, e:
sys.stderr.write("filterator: " + e.message + "\n")
sys.exit(1)
#include <stdio.h>
#include <sys/types.h>
#include <regex.h>
+#include <ctype.h>
#include "shred.h"
static char *c_patterns[] = {
/* Idioms that don't convey any meaning in isolation */
- "return *\\(?[a-z]+\\)? *;", "return *\\(?-?[01]+\\)? *;",
- "goto +[a-z]+;", "exit *\\([01]\\);",
+ "return [a-z]+", "return -?[01]+",
+ "goto +[a-z]+", "exit *\\([01]\\)",
/* Pragmas */
"/\\* *ARGSUSED *\\*/",
"/\\* *NOTREACHED *\\*/",
"/\\* *FALL *THRO?UG?H? *\\*/",
/* Bare C keywords */
- " break ", " case "," continue ",
- " default ", " do ", " else ",
- " enum ", " if ", " goto ",
- " return ", " switch ", " while ",
- " enum ", " int ", " long ",
- " short ", " static ", " struct ",
- " typedef ", " union ", " void ",
- /* Preprocessor constructs */
- "# *define","# *endif","# *else","# *if ", "# *ifdef ","# *ifndef ",
- /* Comment delimiters with no content, and line continuation */
- "/\\*+", "\\*+/", "\\*+", "^ *\\s*\\n", "\\\n",
+ " auto ", " break ", " case "," continue ", " default ", " do ", " else ",
+ " enum ", " if ", " goto "," return ", " switch ", " while ", " int ",
+ " long ", " short ", " static ", " struct ", " typedef ", " union ",
+ " void ",
+ /* Preprocessor constructs (# has already been stripped)*/
+ "^ *define","^ *endif","^ *else", "^ *ifdef ","^ *ifndef ",
/* Common preprocessor macros, not significant by themselves. */
- " ASSERT ",
- " FALSE ",
- " NULL ",
- " STATIC ",
- " TRUE ",
+ " ASSERT ", " EXTERN ", " FALSE ", " NULL "," STATIC ", " TRUE ",
/* Macro include lines are noise, too. */
- "\\s*#include.*","#\\s*line.*",
+ "^ *include.*","^ *line.*",
/* Common error macros. */
" EFAULT ",
" EINVAL ",
};
static regex_t c_regexps[sizeof(c_patterns)/sizeof(*c_patterns)];
-static char c_punct[] = {
- /* Punctuation and whitespace */
- '{', '}', '(', ')', '<', '>', '[', ']',
- '^', '&', '|', '*', '?', '.', '+',
- ';', ':', '%', ',', '-', '/', '=', '!',
- ' ', '\n', '\t', '\v',
-};
-
static char *shell_patterns[] = {
- /* Idioms that don't convey any meaning in isolation */
- "exit *[01];?",
- /* Bare shell keywords */
- " break ", " case ", " done ",
- " do ", " else ", " esac ",
- " fi ", " if ", " return ",
- " shift ", " true ", " while ",
- /* Blank comment */
- "^#\n",
+ " break ", " case ", " done ", " do ", " else ", " esac ", " exit *[01]?",
+ " false ", " fi ", " for", " function", " if ", " return ", " shift ",
+ " true ", "until", " while ",
};
static regex_t shell_regexps[sizeof(shell_patterns)/sizeof(*shell_patterns)];
-static char shell_punct[] = {
- /* Punctuation and whitespace */
- ' ', ':', ';', '\n','\t', '\v',
-};
-
void filter_init(void)
/* initialize line filtering */
{
}
}
-static filtertype active = none;
-static char *punct;
+static unsigned char active = 0;
static regex_t *regexps;
static int nregexps;
-void filter_set(filtertype newfilter)
+void filter_set(int mask)
{
- active = newfilter;
+ active = mask;
- if (newfilter == c)
+ if (mask & C_CODE)
{
regexps = c_regexps;
- punct = c_punct;
nregexps = sizeof(c_regexps)/sizeof(*c_regexps);
}
- else
+ else if (mask & SHELL_CODE)
{
regexps = shell_regexps;
- punct = shell_punct;
nregexps = sizeof(shell_regexps)/sizeof(*shell_regexps);
}
}
int filter_pass(const char *line)
+/* return flags that apply to this line */
{
- if (active == none)
- return(1);
- else if (strspn(line, punct) == strlen(line))
- return(0);
+ if (active == 0)
+ return(SIGNIFICANT);
else
{
char *sp, *tp;
char buf[BUFSIZ];
- int changed;
+ int changed, significant;
/* change all punctuation to spaces */
buf[0] = ' ';
for (sp = (char *)line, tp = buf+1; *sp; sp++)
- if (strchr(punct, *sp) && *sp != ' ')
+ if (ispunct(*sp) || *sp == '\n' || *sp == '\t')
*tp++ = ' ';
else
*tp++ = *sp;
*tp = '\0';
+#ifdef TEST
+ fprintf(stderr, "After removing punctuation: '%s'\n", buf);
+#endif /* TEST */
+
+ if (strspn(line, " ") == strlen(line))
+ return(active);
+
do {
regex_t *re;
for (start = buf+f.rm_so, end = buf+f.rm_eo; *end; end++)
*start++ = *end;
*start = '\0';
+#ifdef TEST
+ fprintf(stderr, "%d...%d -> '%s'\n", f.rm_so, f.rm_eo, buf);
+#endif /* TEST */
changed++;
}
}
} while
(changed);
- return !(buf[0] == '\0' || strspn(buf, " ") == strlen(buf));
+
+
+ significant = !(buf[0] == '\0' || strspn(buf, " ") == strlen(buf));
+
+ return (active | (significant ? SIGNIFICANT : 0));
}
}
char buf[BUFSIZ];
filter_init();
- filter_set(c);
+ filter_set(C_CODE);
while(fgets(buf, BUFSIZ, stdin))
{
- if (filter_pass(buf))
- fputs("yes: ", stdout);
- else
- fputs("no: ", stdout);
+ printf("%02x: ", filter_pass(buf));
fputs(buf, stdout);
}
}
#include <time.h>
#include "shred.h"
-int verbose, debug, minsize;
+int verbose, debug, minsize, nofilter;
struct scf_t
{
static struct filehdr_t dummy_filehdr, *filelist = &dummy_filehdr;
-static int chunk_count, sort_count;
+static int chunk_count, sort_count, dofilter;
static struct hash_t *chunk_buffer;
static struct sorthash_t *sort_buffer;
}
static void write_options(char *buf)
-/* dump a Nirmalization line representing current options */
+/* dump a Normalization line representing current options */
{
buf[0] = '\0';
if (remove_whitespace)
struct hash_t this = np[0];
if (debug)
+ {
fprintf(stderr,
- "%d: %s %s:%d:%d\n",
+ "%d: %s %s:%d:%d",
np-chunk_buffer, hash_dump(this.hash),
*place, this.start, this.end);
+ if (np->flags)
+ {
+ fputc('\t', stderr);
+ dump_flags(np->flags, stderr);
+ fprintf(stderr, " (0x%02x)", np->flags);
+ }
+ fputc('\n', stderr);
+ }
this.start = TONET(this.start);
this.end = TONET(this.end);
fwrite(&this.start, sizeof(linenum_t), 1, ofp);
fwrite(&this.end, sizeof(linenum_t), 1, ofp);
fwrite(&this.hash, sizeof(hashval_t), 1, ofp);
+ fwrite(&this.flags, sizeof(flag_t), 1, ofp);
}
totalchunks += chunk_count;
free(chunk_buffer);
fread(&this.start, sizeof(linenum_t), 1, scf->fp);
fread(&this.end, sizeof(linenum_t), 1, scf->fp);
fread(&this.hash, sizeof(hashval_t), 1, scf->fp);
+ fread(&this.flags, sizeof(flag_t), 1, scf->fp);
this.start = FROMNET(this.start);
this.end = FROMNET(this.end);
corehook(this, filehdr);
}
}
+void dump_flags(const int flags, FILE *fp)
+/* dump tokens corresponding to a flag set */
+{
+ fputc(' ', fp);
+ if (flags & SIGNIFICANT)
+ fputs(" significant", fp);
+ if (flags & C_CODE)
+ fputs(" C", fp);
+ if (flags & SHELL_CODE)
+ fputs(" shell", fp);
+}
+
void dump_array(const char *legend,
struct sorthash_t *obarray, int hashcount)
/* dump the contents of a sort_hash array */
fputs(legend, stdout);
for (np = obarray; np < obarray + hashcount; np++)
- fprintf(stdout, "%2d: %s %s:%d:%d\n",
- np-obarray,
- hash_dump(np->hash.hash),
- np->file->name, np->hash.start, np->hash.end);
+ {
+ fprintf(stdout, "%2d: %s %s:%d:%d",
+ np-obarray,
+ hash_dump(np->hash.hash),
+ np->file->name, np->hash.start, np->hash.end);
+ if (np->hash.flags)
+ {
+ fputc('\t', stdout);
+ dump_flags(np->hash.flags, stdout);
+ fprintf(stdout, " (0x%02x)", np->hash.flags);
+ }
+ fputc('\n', stdout);
+ }
}
void report_time(char *legend, ...)
static void usage(void)
{
- fprintf(stderr,"usage: comparator [-c] [-C] [-d dir ] [-m minsize] [-r] [-o file] [-s shredsize] [-v] [-w] [-x] path...\n");
+ fprintf(stderr,"usage: comparator [-c] [-C] [-d dir ] [-m minsize] [-n] [-r] [-o file] [-s shredsize] [-v] [-w] [-x] path...\n");
fprintf(stderr," -c = generate SCF files\n");
fprintf(stderr," -C = apply C normalizations\n");
fprintf(stderr," -d dir = change directory before digesting.\n");
fprintf(stderr," -m size = set minimum size of span to be output.\n");
+ fprintf(stderr," -n = suppress significance filtering.\n");
fprintf(stderr," -o file = write to the specified file.\n");
fprintf(stderr," -r = renove comments\n");
fprintf(stderr," -s size = set shred size (default %d)\n", shredsize);
struct scf_t *scf;
char *dir, *outfile;
- compile_only = file_only = 0;
+ compile_only = file_only = nofilter = 0;
dir = outfile = NULL;
- while ((status = getopt(argc, argv, "cCd:hm:o:rs:vwx")) != EOF)
+ while ((status = getopt(argc, argv, "cCd:hm:no:rs:vwx")) != EOF)
{
switch (status)
{
minsize = atoi(optarg);
break;
+ case 'n':
+ nofilter = 1;
+ break;
+
case 'o':
outfile = optarg;
break;
usage();
report_time(NULL);
+ filter_init();
/* special case if user gave exactly one tree */
if (!compile_only && argcount == 1)
puts("%%");
for (scf = scflist; scf->next; scf = scf->next)
- printf("%s:%d:%d:%d\n",
+ printf("%s: matches=%d, matchlines=%d, totallines=%d\n",
scf->name,
match_count(scf->name),
line_count(scf->name),
{
p[i].hash.start = min(p[i].hash.start, q[i].hash.start);
p[i].hash.end = max(p[i].hash.end, q[i].hash.end);
+ p[i].hash.flags |= q[i].hash.flags;
}
return(1);
}
* The technique: first mark...
*/
if (SORTHASHCMP(obarray, obarray+1))
- obarray[0].hash.start = UNIQUE_FLAG;
+ obarray[0].hash.flags = UNIQUE_FLAG;
for (np = obarray+1; np < obarray + hashcount-1; np++)
if (SORTHASHCMP(np, np-1) && SORTHASHCMP(np, np+1))
- np->hash.start = UNIQUE_FLAG;
+ np->hash.flags = UNIQUE_FLAG;
if (SORTHASHCMP(obarray+hashcount-2, obarray+hashcount-1))
- obarray[hashcount-1].hash.start = UNIQUE_FLAG;
+ obarray[hashcount-1].hash.flags = UNIQUE_FLAG;
/* ...then sweep. */
for (mp = np = obarray; np < obarray + hashcount; np++)
- if (np->hash.start != UNIQUE_FLAG)
+ if (np->hash.flags != UNIQUE_FLAG)
*mp++ = *np;
/* now we get to reduce the memory footprint */
report_time("Compaction reduced %d shreds to %d",
for (match = hitlist; match < hitlist + mergecount; match++)
{
- int i;
+ int i, flags = 0, maxsize = 0;
- if (minsize)
+ for (i=0; i < match->nmatches; i++)
{
- int maxsize = 0;
-
- for (i=0; i < match->nmatches; i++)
- {
- struct sorthash_t *rp = match->matches+i;
- int matchsize = rp->hash.end - rp->hash.start + 1;
+ struct sorthash_t *rp = match->matches+i;
+ int matchsize = rp->hash.end - rp->hash.start + 1;
- if (matchsize >= maxsize)
- maxsize = matchsize;
- }
- if (maxsize < minsize)
- continue;
+ if (matchsize >= maxsize)
+ maxsize = matchsize;
+ flags |= rp->hash.flags;
}
+ /*
+ * Filter for sigificance right at the last minute. The
+ * point of doing it this way is to allow significance bits to
+ * propagate through range merges.
+ */
+ if (maxsize < minsize || !(nofilter || (flags & SIGNIFICANT)))
+ continue;
+
for (i=0; i < match->nmatches; i++)
{
struct sorthash_t *rp = match->matches+i;
for (match = hitlist; match < hitlist + mergecount; match++)
for (i=0; i < match->nmatches; i++)
- if (strncmp(name, match->matches[i].file->name, strlen(name)))
+ if (!strncmp(name, match->matches[i].file->name, strlen(name)))
{
count += match->matches[i].hash.end - match->matches[i].hash.start + 1;
break;
</varlistentry>
<varlistentry>
-<term><emphasis>Comment</emphasis>: (optional)</term>
-<listitem><para>Arbitrary comment text for human eyes.</para></listitem>
+<term><emphasis>Matches</emphasis>: (mandatory)</term>
+<listitem><para>Count of common segments in the file.</para></listitem>
</varlistentry>
</variablelist>
lengths.</para>
<para>A shred record consists of ushorts (start and end line numbers)
-followed by a fixed-length block of hash data. For the default RXOR
-hash the data is 8 bytes; for MD5, it is 16.</para>
+followed by a fixed-length block of hash data, followed by a flag
+byte. For the default RXOR hash the data is 8 bytes; for MD5, it is
+16.</para>
+
+<para>Defined flags are:</para>
+
+<variablelist>
+<varlistentry>
+<term><emphasis>0x40</emphasis></term>
+<listitem><para>This chunk is deemed significant.</para></listitem>
+</varlistentry>
+<varlistentry>
+<term><emphasis>0x01</emphasis></term>
+<listitem><para>This chunk appears to be C code</para></listitem>
+</varlistentry>
+<varlistentry>
+<term><emphasis>0x02</emphasis></term>
+<listitem><para>This chunk appears to be shell code.</para></listitem>
+</varlistentry>
+</variablelist>
<para>Shreds should be ordered by start line number. Empty files should be
skipped, and nonempty files should always have at least one shred.
comparisons.</para>
</listitem>
</varlistentry>
+</variablelist>
+<para>The data section consists of a tree-properties block followed by
+a sequence of range sets. The tree-properties block consists of any
+number of property lines followed by a %%\n section marker. Each
+property line consists of a tree name, followed by a set of
+comma-separated property settings. Each property setting consists of
+a property, followed by an equal sign, followed by a value. The
+following properties are defined:</para>
+
+<variablelist>
+<varlistentry>
+<term><emphasis>matches</emphasis></term>
+<listitem><para>the count of chunks containing files from the tree</para></listitem>
+</varlistentry>
+<varlistentry>
+<term><emphasis>matchlines</emphasis></term>
+<listitem><para>the count of common lines from the tree</para></listitem>
+</varlistentry>
<varlistentry>
-<term>Comment: (optional)</term>
-<listitem><para>Arbitrary comment text for human eyes.</para></listitem>
+<term><emphasis>totallines</emphasis></term>
+<listitem><para>the total line count of the tree</para></listitem>
</varlistentry>
</variablelist>
-<para>The data section consists of a statistics block followed by a
-sequence of range sets. The statistics block consists of any number
-of count lines followed by a %%\n section marker. Each linecount line
-consists of four colon-separated fields,
-
-<orderedlist>
-<listitem><para>The name of a tree</para></listitem>
-<listitem><para>A count of chunks containing files from the tree</para></listitem>
-<listitem><para>A count of common lines from the tree</para></listitem>
-<listitem><para>The total line count of the tree</para></listitem>
-</orderedlist>
-
-
-
-a tree name and a line count.</para>
-
<para>Each range set consists of two or more ranges followed by a line
containing a %%\n section marker. A range is represented as four
tab-separated fields: filename, start line, end line, and length of
#define FROMNET ntohl
#endif
#define MAX_LINENUM (linenum_t)-1 /* 2s-complement assumption */
-#define UNIQUE_FLAG MAX_LINENUM /* 2s-complement assumption */
/* use this to hold total line count of the entire source tree set */
typedef u_int32_t linecount_t;
+typedef unsigned char flag_t;
+
#include "hash.h"
struct hash_t
{
linenum_t start, end;
hashval_t hash;
+ flag_t flags;
+#define SIGNIFICANT 0x40 /* something in this is not pure syntax */
+#define C_CODE 0x01 /* identified as C code */
+#define SHELL_CODE 0x02 /* identified as shell code */
+#define CATEGORIZED 0x03 /* we can significance-test this */
+#define UNIQUE_FLAG 0x80 /* internal use only */
};
#define SORTHASHCMP(s, t) hash_compare((s)->hash.hash, (t)->hash.hash)
extern int remove_braces;
extern int remove_whitespace;
extern int remove_comments;
-extern int verbose, debug;
+extern int verbose, debug, nofilter;
extern int shredsize, minsize;
/* main.c functions */
extern void extend_current_chunk(void);
extern void dump_array(const char *legend,
struct sorthash_t *obarray, int hashcount);
+extern void dump_flags(const int flags, FILE *fp);
/* shredtree.c functions */
extern char **sorted_file_list(const char *, int *);
void (*hook)(struct hash_t, struct filehdr_t *));
extern void sort_hashes(struct sorthash_t *hashlist, int hashcount);
+/* filter.c functions */
+extern void filter_init(void);
+extern void filter_set(int);
+extern int filter_pass(const char *line);
+
/* shredcompare.c functions */
extern void emit_report1(struct sorthash_t *obarray, int hashcount);
extern void emit_report2(void);
typedef struct
{
char *line;
- int start;
+ linenum_t start;
+ flag_t flags;
}
shred;
struct hash_t out;
/* build completed chunk onto end of array */
+ out.flags = 0;
hash_init();
+ firstline = shredsize;
+ for (i = shredsize - 1; i >= 0; i--)
+ if (display[i].line)
+ firstline = i;
+ firstline = display[firstline].start;
if (debug)
- fprintf(stderr, "Chunk:\n");
+ fprintf(stderr, "Chunk at line %d:\n", firstline);
for (i = 0; i < shredsize; i++)
if (display[i].line)
{
if (debug)
- fprintf(stderr, "%d: '%s'\n", i, display[i].line);
+ fprintf(stderr, "%d (%02x): '%s'\n", i, display[i].flags, display[i].line);
hash_update(display[i].line, strlen(display[i].line));
+ out.flags |= display[i].flags;
}
hash_complete(&out.hash);
- firstline = shredsize;
- for (i = shredsize - 1; i >= 0; i--)
- if (display[i].line)
- firstline = i;
- firstline = display[firstline].start;
out.start = firstline;
out.end = linecount;
exit(1);
}
+ /* deduce what filtering type we should use */
+#define endswith(suff) !strcmp(suff,file->name+strlen(file->name)-strlen(suff))
+ filter_set(0);
+ if (endswith(".c") || endswith(".h"))
+ filter_set(C_CODE);
+ else if (endswith(".sh"))
+ filter_set(SHELL_CODE);
+#undef endswith
+
display = (shred *)calloc(sizeof(shred), shredsize);
accepted = linecount = 0;
}
accepted++;
+ /* maybe we can get the file type from the first line? */
+ if (linecount == 1 && buf[0] == '#')
+ if (strstr(buf, "sh"))
+ filter_set(SHELL_CODE);
+
/* create new shred */
display[shredsize-1].line = strdup(buf);
display[shredsize-1].start = linecount;
+ display[shredsize-1].flags = filter_pass(buf);
/* flush completed chunk */
if (accepted >= shredsize)
for (i=1; i < shredsize; i++)
display[i-1] = display[i];
display[shredsize-1].line = NULL;
+ display[shredsize-1].flags = 0;
}
if (accepted && accepted < shredsize)
hook(emit_chunk(display, linecount), file);
projects / comparator.git / commitdiff