From: Eric S. Raymond Date: Tue, 2 Sep 2003 18:08:29 +0000 (+0000) Subject: We can make an in-core list now. X-Git-Tag: 1.0~61 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=4a1f91956128d7471d0db9a96d17bcaeb0f752eb;p=comparator.git We can make an in-core list now. --- diff --git a/Makefile b/Makefile index 07f2779..6feee81 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -# Makefile for shredcompare tools +# Makefile for comparator tools SOURCES = shredtree.c shredcompare.c main.c md5.c md5.h CFLAGS=-g -all: shredtree +all: comparator main.o: main.c shred.h $(CC) -c $(CFLAGS) main.c @@ -12,27 +12,27 @@ shredtree.o: shredtree.c shred.h $(CC) -c $(CFLAGS) shredtree.c shredcompare.o: shredcompare.c shred.h $(CC) -c $(CFLAGS) shredcompare.c -shredtree: main.o shredtree.o md5.o shredcompare.o - $(CC) main.o shredtree.o md5.o shredcompare.o -o shredtree +comparator: main.o shredtree.o md5.o shredcompare.o + $(CC) main.o shredtree.o md5.o shredcompare.o -o comparator clean: - rm -f shredtree shredcompare shredtree.o md5.o shredcompare.o TEST1 TEST2 - -test-a: shredtree - shredtree -f -c test1 >TEST1 - shredtree -f -c test2 >TEST2 - shredtree TEST1 TEST2 -test-b: shredtree - shredtree -f -c test3 >TEST1 - shredtree -f -c test4 >TEST2 - shredtree TEST1 TEST2 - -goodtest: shredtree - shredtree -f -c -d ~/src/unix src5r4/src/ucbhead >src5r4.shif - shredtree -f -c -d ~/src/unix linux-2.6.0-test4/include/linux >linux-2.6.0-test4.shif - shredtree src5r4.shif linux-2.6.0-test4.shif - -bigtest: shredtree - shredtree -f -c -d ~/src/unix src5r4 >src5r4.shif - shredtree -f -c -d ~/src/unix linux-2.6.0-test4 >linux-2.6.0-test4.shif - shredtree src5r4.shif linux-2.6.0-test4.shif >TEST.LOG + rm -f comparator shredtree.o md5.o shredcompare.o TEST1 TEST2 + +test-a: comparator + comparator -f -c test1 >TEST1 + comparator -f -c test2 >TEST2 + comparator TEST1 TEST2 +test-b: comparator + comparator -f -c test3 >TEST1 + comparator -f -c test4 >TEST2 + comparator TEST1 TEST2 + +goodtest: comparator + comparator -f -c -d ~/src/unix src5r4/src/ucbhead >src5r4.shif + comparator -f -c -d ~/src/unix linux-2.6.0-test4/include/linux >linux-2.6.0-test4.shif + comparator src5r4.shif linux-2.6.0-test4.shif + +bigtest: comparator + comparator -f -c -d ~/src/unix src5r4 >src5r4.shif + comparator -f -c -d ~/src/unix linux-2.6.0-test4 >linux-2.6.0-test4.shif + comparator src5r4.shif linux-2.6.0-test4.shif >TEST.LOG diff --git a/report.c b/report.c index 587b9cd..acb9281 100644 --- a/report.c +++ b/report.c @@ -32,11 +32,6 @@ struct shif_t }; static struct shif_t *shiflist; -struct sorthash_t -{ - struct hash_t hash; - char *file; -}; #define HASHCMP(s, t) memcmp((s)->hash.hash, (t)->hash.hash, HASHSIZE) static struct sorthash_t *obarray, *np; diff --git a/shred.h b/shred.h index 1285c22..189f07b 100644 --- a/shred.h +++ b/shred.h @@ -23,6 +23,12 @@ struct hash_t unsigned char hash[HASHSIZE]; }; +struct sorthash_t +{ + struct hash_t hash; + char *file; +}; + /* control bits, meant to be set at startup */ extern int c_only; extern int rws; diff --git a/shredtree.c b/shredtree.c index 99d2a92..85c9527 100644 --- a/shredtree.c +++ b/shredtree.c @@ -16,9 +16,10 @@ int rws = 0; int debug = 0; int shredsize = 5; -/* globally visible data; chunk_buffer should be a malloc area set by the caller */ -int file_count, chunk_count; +/* globally visible data */ +int file_count, chunk_count, sort_count; struct hash_t *chunk_buffer; +struct sorthash_t *sort_buffer; struct item { @@ -62,12 +63,13 @@ typedef struct } shred; -static void emit_chunk(shred *display, int linecount) -/* emit chunk corresponding to current display; modifies globals chunk_buffer and chunk_count */ +static struct hash_t emit_chunk(shred *display, int linecount) +/* emit chunk corresponding to current display */ { struct md5_ctx ctx; int i, firstline; unsigned char *cp; + struct hash_t out; if (debug) { @@ -87,26 +89,54 @@ static void emit_chunk(shred *display, int linecount) } } - chunk_buffer = (struct hash_t *)realloc(chunk_buffer, - sizeof(struct hash_t) * (chunk_count+1)); - /* build completed chunk onto end of array */ md5_init_ctx(&ctx); for (i = 0; i < shredsize; i++) if (display[i].line) md5_process_bytes(display[i].line, strlen(display[i].line), &ctx); - md5_finish_ctx(&ctx, (void *)&chunk_buffer[chunk_count].hash); + md5_finish_ctx(&ctx, (void *)&out.hash); for (i = shredsize - 1; i >= 0; i--) if (display[i].line) firstline = i; firstline = display[firstline].number; - chunk_buffer[chunk_count].start = TONET(firstline); - chunk_buffer[chunk_count].end = TONET(linecount); - chunk_count++; + out.start = TONET(firstline); + out.end = TONET(linecount); + + return(out); +} + +/* + * We differentiate these two functions in order to avaid the memory + * overhead of the extra (char *) when assembling shred lists to be written + * to a file. This is only worth bothering with because the data sets + * can get quite large. + */ + +static void filehook(struct hash_t hash, const char *file) +/* hook to store only hashes */ +{ + chunk_buffer = (struct hash_t *)realloc(chunk_buffer, + sizeof(struct hash_t) * (chunk_count+1)); + + chunk_buffer[chunk_count++] = hash; +} + + +static void corehook(struct hash_t hash, const char *file) +/* hook to store hash and file */ +{ + sort_buffer = (struct sorthash_t *)realloc(sort_buffer, + sizeof(struct sorthash_t) * (sort_count+1)); + + sort_buffer[sort_count].hash = hash; + sort_buffer[sort_count].file = (char *)file; + sort_count++; } -static void shredfile(const char *file) -/* emit hash section for specified file; modifies globals chunk_buffer and chunk_count */ + +static void shredfile(const char *file, + void (*hook)(struct hash_t, const char *)) +/* emit hash section for specified file */ { FILE *fp; char buf[BUFSIZ]; @@ -136,7 +166,7 @@ static void shredfile(const char *file) /* flush completed chunk */ if (accepted >= shredsize) - emit_chunk(display, linecount); + hook(emit_chunk(display, linecount), file); /* shreds in progress are shifted down */ free(display[0].line); @@ -145,7 +175,7 @@ static void shredfile(const char *file) display[shredsize-1].line = NULL; } if (linecount < shredsize) - emit_chunk(display, linecount); + hook(emit_chunk(display, linecount), file); free(display); fclose(fp); @@ -173,18 +203,10 @@ static int stringsort(void *a, void *b) return strcmp(*(char **)a, *(char **)b); } -void generate_shredfile(const char *tree, FILE *ofp) -/* generate shred file for given tree */ +static char **sorted_file_list(const char *tree) +/* generate a sorted list of files under the given tree */ { char **place, **list; - u_int32_t netfile_count; - - fputs("#SHIF-A 1.0\n", ofp); - fputs("Generator-Program: shredtree 1.0\n", ofp); - fputs("Hash-Method: MD5\n", ofp); - fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none"); - fprintf(ofp, "Shred-Size: %d\n", shredsize); - fputs("%%\n", ofp); /* make file list */ ftw(tree, treewalker, 16); @@ -196,8 +218,23 @@ void generate_shredfile(const char *tree, FILE *ofp) /* the objective -- sort */ qsort(list, file_count, sizeof(char *), stringsort); +} + +void generate_shredfile(const char *tree, FILE *ofp) +/* generate shred file for given tree */ +{ + char **place, **list; + u_int32_t netfile_count; + + list = sorted_file_list(tree); + + fputs("#SHIF-A 1.0\n", ofp); + fputs("Generator-Program: shredtree 1.0\n", ofp); + fputs("Hash-Method: MD5\n", ofp); + fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none"); + fprintf(ofp, "Shred-Size: %d\n", shredsize); + fputs("%%\n", ofp); - /* generate the list */ netfile_count = htonl(file_count); fwrite(&netfile_count, sizeof(u_int32_t), 1, ofp); for (place = list; place < list + file_count; place++) @@ -207,7 +244,7 @@ void generate_shredfile(const char *tree, FILE *ofp) /* shredfile adds data to chunk_buffer */ chunk_buffer = (struct hash_t *)calloc(sizeof(struct hash_t), 1); chunk_count = 0; - shredfile(*place); + shredfile(*place, filehook); /* the actual output */ fputs(*place, ofp); @@ -219,4 +256,20 @@ void generate_shredfile(const char *tree, FILE *ofp) } } +struct sorthash_t *generate_shredlist(const char *tree) +/* generate an in-core list of sorthash structures */ +{ + char **place, **list; + + sort_buffer = (struct sorthash_t*)calloc(sizeof(struct sorthash_t),1); + sort_count = 0; + + list = sorted_file_list(tree); + for (place = list; place < list + file_count; place++) + shredfile(*place, corehook); + + /* caller is responsible for freeing this */ + return(sort_buffer); +} + /* shredtree.c ends here */