From: Eric S. Raymond Date: Fri, 12 Sep 2003 09:28:29 +0000 (+0000) Subject: We have a statistics section. X-Git-Tag: 1.4~6 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=0fd94ec93686dfa8a1c36c1163d5a1eb59920a3a;p=comparator.git We have a statistics section. --- diff --git a/Makefile b/Makefile index 35861e2..58dc325 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ DOCS = README NEWS comparator.xml scf-standard.xml COPYING EXTRAS = shredtree.py shredcompare.py TEST = test SOURCES = $(CODE) $(DOCS) $(EXTRAS) $(TEST) comparator.spec Makefile -CFLAGS = -O +CFLAGS = -g LDFLAGS = all: comparator comparator.1 main.o: main.c shred.h - $(CC) -c $(CFLAGS) main.c + $(CC) -DVERSION=\"$(VERS)\" -c $(CFLAGS) main.c shredtree.o: shredtree.c shred.h $(CC) -c $(CFLAGS) shredtree.c report.o: report.c shred.h diff --git a/comparator.xml b/comparator.xml index 8eb58e9..c88c03f 100644 --- a/comparator.xml +++ b/comparator.xml @@ -10,7 +10,7 @@ comparator filterator -fast comparisons among large source trees +fast location of common code in large source trees @@ -40,10 +40,10 @@ comparator is a program for quickly finding common sections in two or more source-code trees. It's named after a now-obsolete astronomical instrument called a blink comparator -that was used to find moving objects in starfield photos. -filterator is a postprocessor for -comparator reports that can apply -significance filtering. +that was used to find moving objects in starfield photos. These +reports tend to be noisy; filterator is a +postprocessor for comparator reports that +can apply significance filtering. comparator works by first chopping the specified trees into overlapping shreds (by default 3 lines long) @@ -179,10 +179,13 @@ command-line options you have in effect for shredding source-code trees. The program does some sanity checking but will not prevent you from shooting yourself in the foot. -The code for finding duplicate hashes uses a brute-force -approach that relies on being able to allocate core for the entire -hash list generated during processing. The results are almost -ridiculously fast on machines with enough core (the author has +The implementation is O(n log +n) in the size of the code trees and +O(n2) in the number of +common segments. The code for finding duplicate hashes uses a +brute-force approach that relies on being able to allocate core for +the entire hash list generated during processing. The results are +almost ridiculously fast on machines with enough core (the author has observed speeds of up to two million lines of code compared per minute on a 1.8GHz Intel box), but machines with small memories will thrash their guts out as the VM subsystem tries to cope. diff --git a/filterator b/filterator index a154425..2d82916 100755 --- a/filterator +++ b/filterator @@ -146,6 +146,7 @@ if __name__ == '__main__': sys.exit(2) nofilter = debug = False minsize = 0 + linecounts = {} for (opt, val) in optlist: if opt == '-d': os.chdir(val) @@ -163,6 +164,7 @@ if __name__ == '__main__': if not id.startswith("#SCF-B "): sys.stderr.write("filterator: input is not a SCF-B file.\n") sys.exit(1) + # Read metadata while True: line = sys.stdin.readline() if not line or line == '%%\n': @@ -177,5 +179,12 @@ if __name__ == '__main__': merge_program = value elif tag == "Hash-Method": hash_method = value + # Read file statistics + while True: + line = sys.stdin.readline() + if not line or line == '%%\n': + break + (tag, value) = line.split(":") + linecounts[tag] = int(value.strip()) report_common() diff --git a/main.c b/main.c index bd0e808..a6063ce 100644 --- a/main.c +++ b/main.c @@ -242,15 +242,15 @@ static int merge_tree(char *tree) return(totallines); } -static void init_scf(char *file, struct scf_t *scf) +static void init_scf(char *file, struct scf_t *scf, const int readfile) /* add to the in-core list of sorthash structures from a SCF file */ { - if (file) + scf->name = strdup(file); + if (readfile) { char buf[BUFSIZ]; /* read in the SCF metadata block and add it to the in-core list */ - scf->name = strdup(file); scf->fp = fopen(scf->name, "r"); fgets(buf, sizeof(buf), scf->fp); if (strncmp(buf, "#SCF-A 1.1", 9)) @@ -477,7 +477,7 @@ main(int argc, char *argv[]) olddir = getcwd(NULL, 0); /* may fail off Linux */ chdir(dir); } - init_scf(source, scf); + init_scf(source, scf, 1); if (dir) chdir(olddir); } @@ -513,8 +513,8 @@ main(int argc, char *argv[]) olddir = getcwd(NULL, 0); /* may fail off Linux */ chdir(dir); } - init_scf(NULL, scf); - merge_tree(source); + init_scf(source, scf, 0); + scf->totallines = merge_tree(source); if (dir) chdir(olddir); } @@ -567,12 +567,15 @@ main(int argc, char *argv[]) } /* now we're ready to emit the report */ - puts("#SCF-B 1.0"); + puts("#SCF-B 1.1"); printf("Hash-Method: %s\n", scflist->hash_method); puts("Merge-Program: comparator 1.0"); printf("Normalization: %s\n", scflist->normalization); printf("Shred-Size: %d\n", scflist->shred_size); puts("%%"); + for (scf = scflist; scf->next; scf = scf->next) + printf("%s:%d\n", scf->name, scf->totallines); + puts("%%"); report_time("Hash merge done, %d shreds", sort_count); sort_hashes(sort_buffer, sort_count); diff --git a/scf-standard.xml b/scf-standard.xml index 1bd2cb1..f0a7bf6 100644 --- a/scf-standard.xml +++ b/scf-standard.xml @@ -18,6 +18,14 @@ + + 1.3 + 2003-12-07 + esr + + SCF format goes to 1.1; line count is added. + + 1.2 2003-11-07 @@ -26,7 +34,7 @@ Added remove-braces. - + 1.1 2003-10-07 esr @@ -179,12 +187,13 @@ able to describe what reduction and filtering steps were performed. these needs. These formats will be known as SCF (source Comparison Interchange Format) files, pronounced "skiff". -Each format consists of three sections: an identification line, -metadata, and data. The identification line consists of a hash sign, -a format identifier, a space, and a revision level. The metadata section -consists of lines in the format of an RFC-822 header (a tag, followed by -a colon and whitespace, followed by a value). The data section consists -of the line "%%\n" followed by data. Metadata is encoded in UTF-8. +Each format consists of four sections: an identification line, +metadata, hash data, and (optionally) a statistics trailer. The +identification line consists of a hash sign, a format identifier, a +space, and a revision level. The metadata section consists of lines +in the format of an RFC-822 header (a tag, followed by a colon and +whitespace, followed by a value). The data section consists of the +line "%%\n" followed by data. Metadata is encoded in UTF-8. For each format, we describe the format identifier, the metadata tags and their semantics, and the format of the data section. Some metadata @@ -197,7 +206,7 @@ operations like diff(1) or comm(1) can give meaningful results. SCF-A: This is the format for hash lists. The identifier is 'SCF-A'. -At version 1.0, the following metadata tags are defined: +At version 1.1, the following metadata tags are defined: @@ -254,7 +263,8 @@ may include the following: order, followed by a series of file sections. Each section consists of a filename line, terminated by a newline, followed by a count of shreds expressed as a four-byte integer in network byte order, -followed by a sequence of shred records. +followed by a sequence of shred records. The length of the data +section is drtivable from the data lengths. A shred record consists of two integers in network byte order (start and end line numbers) followed by a fixed-length block of hash @@ -264,6 +274,10 @@ data. For the default MD5 hash the data is 16 bytes. skipped, and nonempty files should always have at least one shred. Files should be listed in sort order by name. +This format has a statistics trailer. At level 1.1 it consists +of a single 4-byte unsigned integer in network order, a count of lines +for the associated source tree. + This unpleasant binary format has been chosen to make processing as fast as possible, because shred data sets are quite large in holding down I/O is an important consideration. @@ -320,6 +334,8 @@ start line, then by end line. Ranges within range sets should be sorted by natural order. Range sets should be sorted by the natural order of their first member. +This format has no statistics trailer. + It is strongly recommended that when a SCF-B file is generated from SCF-A files, the Normalization and Shred-Size headers should be copied into the result. diff --git a/shred.h b/shred.h index e111031..0ab16b6 100644 --- a/shred.h +++ b/shred.h @@ -39,7 +39,7 @@ extern void corehook(struct hash_t hash, const char *file); /* shredtree.c functions */ extern char **sorted_file_list(const char *, int *); -extern void shredfile(const char *, void (*hook)(struct hash_t, const char *)); +extern int shredfile(const char *, void (*hook)(struct hash_t, const char *)); extern void sort_hashes(struct sorthash_t *hashlist, int hashcount); /* shredcompare.c functions */ diff --git a/shredtree.c b/shredtree.c index b361eb7..8a4de6d 100644 --- a/shredtree.c +++ b/shredtree.c @@ -156,7 +156,7 @@ static struct hash_t emit_chunk(shred *display, int linecount) return(out); } -void shredfile(const char *file, +int shredfile(const char *file, void (*hook)(struct hash_t, const char *)) /* emit hash section for specified file */ { @@ -201,6 +201,7 @@ void shredfile(const char *file, free(display); fclose(fp); + return(linecount); } /*************************************************************************