EXTRAS = shredtree.py shredcompare.py
TEST = test
SOURCES = $(CODE) $(DOCS) $(EXTRAS) $(TEST) comparator.spec Makefile
-CFLAGS = -O
+CFLAGS = -g
LDFLAGS =
all: comparator comparator.1
main.o: main.c shred.h
- $(CC) -c $(CFLAGS) main.c
+ $(CC) -DVERSION=\"$(VERS)\" -c $(CFLAGS) main.c
shredtree.o: shredtree.c shred.h
$(CC) -c $(CFLAGS) shredtree.c
report.o: report.c shred.h
<refnamediv id='name'>
<refname>comparator</refname>
<refname>filterator</refname>
-<refpurpose>fast comparisons among large source trees</refpurpose>
+<refpurpose>fast location of common code in large source trees</refpurpose>
</refnamediv>
<refsynopsisdiv id='synopsis'>
<para><application>comparator</application> is a program for quickly
finding common sections in two or more source-code trees. It's named
after a now-obsolete astronomical instrument called a blink comparator
-that was used to find moving objects in starfield photos.
-<application>filterator</application> is a postprocessor for
-<application>comparator</application> reports that can apply
-significance filtering.</para>
+that was used to find moving objects in starfield photos. These
+reports tend to be noisy; <application>filterator</application> is a
+postprocessor for <application>comparator</application> reports that
+can apply significance filtering.</para>
<para><application>comparator</application> works by first chopping
the specified trees into overlapping shreds (by default 3 lines long)
trees. The program does some sanity checking but will not prevent you
from shooting yourself in the foot.</para>
-<para>The code for finding duplicate hashes uses a brute-force
-approach that relies on being able to allocate core for the entire
-hash list generated during processing. The results are almost
-ridiculously fast on machines with enough core (the author has
+<para>The implementation is O(<emphasis>n</emphasis> log
+<emphasis>n</emphasis>) in the size of the code trees and
+O(<emphasis>n</emphasis><superscript>2</superscript>) in the number of
+common segments. The code for finding duplicate hashes uses a
+brute-force approach that relies on being able to allocate core for
+the entire hash list generated during processing. The results are
+almost ridiculously fast on machines with enough core (the author has
observed speeds of up to two million lines of code compared per minute
on a 1.8GHz Intel box), but machines with small memories will thrash
their guts out as the VM subsystem tries to cope.</para>
sys.exit(2)
nofilter = debug = False
minsize = 0
+ linecounts = {}
for (opt, val) in optlist:
if opt == '-d':
os.chdir(val)
if not id.startswith("#SCF-B "):
sys.stderr.write("filterator: input is not a SCF-B file.\n")
sys.exit(1)
+ # Read metadata
while True:
line = sys.stdin.readline()
if not line or line == '%%\n':
merge_program = value
elif tag == "Hash-Method":
hash_method = value
+ # Read file statistics
+ while True:
+ line = sys.stdin.readline()
+ if not line or line == '%%\n':
+ break
+ (tag, value) = line.split(":")
+ linecounts[tag] = int(value.strip())
report_common()
return(totallines);
}
-static void init_scf(char *file, struct scf_t *scf)
+static void init_scf(char *file, struct scf_t *scf, const int readfile)
/* add to the in-core list of sorthash structures from a SCF file */
{
- if (file)
+ scf->name = strdup(file);
+ if (readfile)
{
char buf[BUFSIZ];
/* read in the SCF metadata block and add it to the in-core list */
- scf->name = strdup(file);
scf->fp = fopen(scf->name, "r");
fgets(buf, sizeof(buf), scf->fp);
if (strncmp(buf, "#SCF-A 1.1", 9))
olddir = getcwd(NULL, 0); /* may fail off Linux */
chdir(dir);
}
- init_scf(source, scf);
+ init_scf(source, scf, 1);
if (dir)
chdir(olddir);
}
olddir = getcwd(NULL, 0); /* may fail off Linux */
chdir(dir);
}
- init_scf(NULL, scf);
- merge_tree(source);
+ init_scf(source, scf, 0);
+ scf->totallines = merge_tree(source);
if (dir)
chdir(olddir);
}
}
/* now we're ready to emit the report */
- puts("#SCF-B 1.0");
+ puts("#SCF-B 1.1");
printf("Hash-Method: %s\n", scflist->hash_method);
puts("Merge-Program: comparator 1.0");
printf("Normalization: %s\n", scflist->normalization);
printf("Shred-Size: %d\n", scflist->shred_size);
puts("%%");
+ for (scf = scflist; scf->next; scf = scf->next)
+ printf("%s:%d\n", scf->name, scf->totallines);
+ puts("%%");
report_time("Hash merge done, %d shreds", sort_count);
sort_hashes(sort_buffer, sort_count);
</author>
<revhistory>
+ <revision>
+ <revnumber>1.3</revnumber>
+ <date>2003-12-07</date>
+ <authorinitials>esr</authorinitials>
+ <revremark>
+ SCF format goes to 1.1; line count is added.
+ </revremark>
+ </revision>
<revision>
<revnumber>1.2</revnumber>
<date>2003-11-07</date>
Added remove-braces.
</revremark>
</revision>
- <revision>
+ <revision>
<revnumber>1.1</revnumber>
<date>2003-10-07</date>
<authorinitials>esr</authorinitials>
these needs. These formats will be known as SCF (source Comparison
Interchange Format) files, pronounced "skiff".</para>
-<para>Each format consists of three sections: an identification line,
-metadata, and data. The identification line consists of a hash sign,
-a format identifier, a space, and a revision level. The metadata section
-consists of lines in the format of an RFC-822 header (a tag, followed by
-a colon and whitespace, followed by a value). The data section consists
-of the line "%%\n" followed by data. Metadata is encoded in UTF-8.</para>
+<para>Each format consists of four sections: an identification line,
+metadata, hash data, and (optionally) a statistics trailer. The
+identification line consists of a hash sign, a format identifier, a
+space, and a revision level. The metadata section consists of lines
+in the format of an RFC-822 header (a tag, followed by a colon and
+whitespace, followed by a value). The data section consists of the
+line "%%\n" followed by data. Metadata is encoded in UTF-8.</para>
<para>For each format, we describe the format identifier, the metadata tags
and their semantics, and the format of the data section. Some metadata
<sect2><title>SCF-A:</title>
<para>This is the format for hash lists. The identifier is 'SCF-A'.
-At version 1.0, the following metadata tags are defined:</para>
+At version 1.1, the following metadata tags are defined:</para>
<variablelist>
<varlistentry>
order, followed by a series of file sections. Each section consists
of a filename line, terminated by a newline, followed by a count of
shreds expressed as a four-byte integer in network byte order,
-followed by a sequence of shred records.</para>
+followed by a sequence of shred records. The length of the data
+section is drtivable from the data lengths.</para>
<para>A shred record consists of two integers in network byte order
(start and end line numbers) followed by a fixed-length block of hash
skipped, and nonempty files should always have at least one shred.
Files should be listed in sort order by name.</para>
+<para>This format has a statistics trailer. At level 1.1 it consists
+of a single 4-byte unsigned integer in network order, a count of lines
+for the associated source tree.</para>
+
<para>This unpleasant binary format has been chosen to make processing as
fast as possible, because shred data sets are quite large in holding
down I/O is an important consideration.</para>
sorted by natural order. Range sets should be sorted by the natural
order of their first member.</para>
+<para>This format has no statistics trailer.</para>
+
<para>It is strongly recommended that when a SCF-B file is generated from
SCF-A files, the Normalization and Shred-Size headers should be copied
into the result.</para>
/* shredtree.c functions */
extern char **sorted_file_list(const char *, int *);
-extern void shredfile(const char *, void (*hook)(struct hash_t, const char *));
+extern int shredfile(const char *, void (*hook)(struct hash_t, const char *));
extern void sort_hashes(struct sorthash_t *hashlist, int hashcount);
/* shredcompare.c functions */
return(out);
}
-void shredfile(const char *file,
+int shredfile(const char *file,
void (*hook)(struct hash_t, const char *))
/* emit hash section for specified file */
{
free(display);
fclose(fp);
+ return(linecount);
}
/*************************************************************************
projects / comparator.git / commitdiff