We can make an in-core list now.

diff --git a/Makefile b/Makefile
index 07f277916779879bbdc68702bf02478339215714..6feee819c0bdb237406809910255e248be32586b 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,10 @@
-# Makefile for shredcompare tools
+# Makefile for comparator tools
 
 SOURCES = shredtree.c shredcompare.c main.c md5.c md5.h
 
 CFLAGS=-g
 
-all: shredtree
+all: comparator
 
 main.o: main.c shred.h
        $(CC) -c $(CFLAGS) main.c 
@@ -12,27 +12,27 @@ shredtree.o: shredtree.c shred.h
        $(CC) -c $(CFLAGS) shredtree.c 
 shredcompare.o: shredcompare.c shred.h
        $(CC) -c $(CFLAGS) shredcompare.c 
-shredtree: main.o shredtree.o md5.o shredcompare.o
-       $(CC) main.o shredtree.o md5.o shredcompare.o -o shredtree
+comparator: main.o shredtree.o md5.o shredcompare.o
+       $(CC) main.o shredtree.o md5.o shredcompare.o -o comparator
 
 clean:
-       rm -f shredtree shredcompare shredtree.o md5.o shredcompare.o TEST1 TEST2
-
-test-a: shredtree
-       shredtree -f -c test1 >TEST1
-       shredtree -f -c test2 >TEST2
-       shredtree TEST1 TEST2
-test-b: shredtree
-       shredtree -f -c test3 >TEST1
-       shredtree -f -c test4 >TEST2
-       shredtree TEST1 TEST2
-
-goodtest: shredtree
-       shredtree -f -c -d ~/src/unix src5r4/src/ucbhead >src5r4.shif
-       shredtree -f -c -d ~/src/unix linux-2.6.0-test4/include/linux >linux-2.6.0-test4.shif
-       shredtree src5r4.shif linux-2.6.0-test4.shif
-
-bigtest: shredtree
-       shredtree -f -c -d ~/src/unix src5r4 >src5r4.shif
-       shredtree -f -c -d ~/src/unix linux-2.6.0-test4 >linux-2.6.0-test4.shif
-       shredtree src5r4.shif linux-2.6.0-test4.shif >TEST.LOG
+       rm -f comparator shredtree.o md5.o shredcompare.o TEST1 TEST2
+
+test-a: comparator
+       comparator -f -c test1 >TEST1
+       comparator -f -c test2 >TEST2
+       comparator TEST1 TEST2
+test-b: comparator
+       comparator -f -c test3 >TEST1
+       comparator -f -c test4 >TEST2
+       comparator TEST1 TEST2
+
+goodtest: comparator
+       comparator -f -c -d ~/src/unix src5r4/src/ucbhead >src5r4.shif
+       comparator -f -c -d ~/src/unix linux-2.6.0-test4/include/linux >linux-2.6.0-test4.shif
+       comparator src5r4.shif linux-2.6.0-test4.shif
+
+bigtest: comparator
+       comparator -f -c -d ~/src/unix src5r4 >src5r4.shif
+       comparator -f -c -d ~/src/unix linux-2.6.0-test4 >linux-2.6.0-test4.shif
+       comparator src5r4.shif linux-2.6.0-test4.shif >TEST.LOG

diff --git a/report.c b/report.c
index 587b9cdfac848acc0dbd6296df129b76b90400a9..acb9281f8d83dd13f22053b496ca342ab6fdbc7e 100644 (file)
--- a/report.c
+++ b/report.c
@@ -32,11 +32,6 @@ struct shif_t
 };
 static struct shif_t *shiflist;
 
-struct sorthash_t
-{
-    struct hash_t      hash;
-    char               *file;
-};
 #define HASHCMP(s, t)  memcmp((s)->hash.hash, (t)->hash.hash, HASHSIZE)
 
 static struct sorthash_t *obarray, *np;

diff --git a/shred.h b/shred.h
index 1285c2213a500a9ac129dfb5ffbe09e50a79f83a..189f07bf3bce383fa66ec1a916cc18366c8eeef8 100644 (file)
--- a/shred.h
+++ b/shred.h
@@ -23,6 +23,12 @@ struct hash_t
     unsigned char      hash[HASHSIZE];
 };
 
+struct sorthash_t
+{
+    struct hash_t      hash;
+    char               *file;
+};
+
 /* control bits, meant to be set at startup */
 extern int c_only;
 extern int rws;

diff --git a/shredtree.c b/shredtree.c
index 99d2a92d64eaeda010c363e9d79565260e000698..85c9527518fdb45198d90e3b1a440c2deaa2e581 100644 (file)
--- a/shredtree.c
+++ b/shredtree.c
@@ -16,9 +16,10 @@ int rws = 0;
 int debug = 0;
 int shredsize = 5;
 
-/* globally visible data; chunk_buffer should be a malloc area set by the caller */
-int file_count, chunk_count;
+/* globally visible data */
+int file_count, chunk_count, sort_count;
 struct hash_t *chunk_buffer;
+struct sorthash_t *sort_buffer;
 
 struct item
 {
@@ -62,12 +63,13 @@ typedef struct
 }
 shred;
 
-static void emit_chunk(shred *display, int linecount)
-/* emit chunk corresponding to current display; modifies globals chunk_buffer and chunk_count */
+static struct hash_t emit_chunk(shred *display, int linecount) 
+/* emit chunk corresponding to current display */
 {
     struct md5_ctx     ctx;
     int                i, firstline;
     unsigned char      *cp;
+    struct hash_t      out;
 
     if (debug)
     {
@@ -87,26 +89,54 @@ static void emit_chunk(shred *display, int linecount)
            }    
     }
 
-    chunk_buffer = (struct hash_t *)realloc(chunk_buffer, 
-                                     sizeof(struct hash_t) * (chunk_count+1));
-
     /* build completed chunk onto end of array */
     md5_init_ctx(&ctx);
     for (i = 0; i < shredsize; i++)
        if (display[i].line)
            md5_process_bytes(display[i].line, strlen(display[i].line), &ctx);
-    md5_finish_ctx(&ctx, (void *)&chunk_buffer[chunk_count].hash);
+    md5_finish_ctx(&ctx, (void *)&out.hash);
     for (i = shredsize - 1; i >= 0; i--)
        if (display[i].line)
            firstline = i;
     firstline = display[firstline].number;
-    chunk_buffer[chunk_count].start = TONET(firstline);
-    chunk_buffer[chunk_count].end = TONET(linecount);
-    chunk_count++;
+    out.start = TONET(firstline);
+    out.end = TONET(linecount);
+
+    return(out);
+}
+
+/* 
+ * We differentiate these two functions in order to avaid the memory
+ * overhead of the extra (char *) when assembling shred lists to be written
+ * to a file.  This is only worth bothering with because the data sets
+ * can get quite large.
+ */
+
+static void filehook(struct hash_t hash, const char *file)
+/* hook to store only hashes */
+{
+    chunk_buffer = (struct hash_t *)realloc(chunk_buffer, 
+                                     sizeof(struct hash_t) * (chunk_count+1));
+    
+    chunk_buffer[chunk_count++] = hash;
+}
+
+
+static void corehook(struct hash_t hash, const char *file)
+/* hook to store hash and file */
+{
+    sort_buffer = (struct sorthash_t *)realloc(sort_buffer, 
+                                     sizeof(struct sorthash_t) * (sort_count+1));
+    
+    sort_buffer[sort_count].hash = hash;
+    sort_buffer[sort_count].file = (char *)file;
+    sort_count++;
 }
 
-static void shredfile(const char *file)
-/* emit hash section for specified file; modifies globals chunk_buffer and chunk_count */
+
+static void shredfile(const char *file, 
+                     void (*hook)(struct hash_t, const char *))
+/* emit hash section for specified file */
 {
     FILE *fp;
     char buf[BUFSIZ];
@@ -136,7 +166,7 @@ static void shredfile(const char *file)
 
        /* flush completed chunk */
        if (accepted >= shredsize)
-           emit_chunk(display, linecount);
+           hook(emit_chunk(display, linecount), file);
 
        /* shreds in progress are shifted down */
        free(display[0].line);
@@ -145,7 +175,7 @@ static void shredfile(const char *file)
        display[shredsize-1].line = NULL;
     }
     if (linecount < shredsize)
-       emit_chunk(display, linecount);
+       hook(emit_chunk(display, linecount), file);
 
     free(display);
     fclose(fp);
@@ -173,18 +203,10 @@ static int stringsort(void *a, void *b)
     return strcmp(*(char **)a, *(char **)b);
 }
 
-void generate_shredfile(const char *tree, FILE *ofp)
-/* generate shred file for given tree */
+static char **sorted_file_list(const char *tree)
+/* generate a sorted list of files under the given tree */
 {
     char       **place, **list;
-    u_int32_t  netfile_count;
-
-    fputs("#SHIF-A 1.0\n", ofp);
-    fputs("Generator-Program: shredtree 1.0\n", ofp);
-    fputs("Hash-Method: MD5\n", ofp);
-    fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none");
-    fprintf(ofp, "Shred-Size: %d\n", shredsize);
-    fputs("%%\n", ofp);
 
     /* make file list */
     ftw(tree, treewalker, 16);
@@ -196,8 +218,23 @@ void generate_shredfile(const char *tree, FILE *ofp)
 
     /* the objective -- sort */
     qsort(list, file_count, sizeof(char *), stringsort);
+}
+
+void generate_shredfile(const char *tree, FILE *ofp)
+/* generate shred file for given tree */
+{
+    char       **place, **list;
+    u_int32_t  netfile_count;
+
+    list = sorted_file_list(tree);
+
+    fputs("#SHIF-A 1.0\n", ofp);
+    fputs("Generator-Program: shredtree 1.0\n", ofp);
+    fputs("Hash-Method: MD5\n", ofp);
+    fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none");
+    fprintf(ofp, "Shred-Size: %d\n", shredsize);
+    fputs("%%\n", ofp);
 
-    /* generate the list */
     netfile_count = htonl(file_count);
     fwrite(&netfile_count, sizeof(u_int32_t), 1, ofp);
     for (place = list; place < list + file_count; place++)
@@ -207,7 +244,7 @@ void generate_shredfile(const char *tree, FILE *ofp)
        /* shredfile adds data to chunk_buffer */
        chunk_buffer = (struct hash_t *)calloc(sizeof(struct hash_t), 1);
        chunk_count = 0;
-       shredfile(*place);
+       shredfile(*place, filehook);
 
        /* the actual output */
        fputs(*place, ofp);
@@ -219,4 +256,20 @@ void generate_shredfile(const char *tree, FILE *ofp)
     }
 }
 
+struct sorthash_t *generate_shredlist(const char *tree)
+/* generate an in-core list of sorthash structures */
+{
+    char       **place, **list;
+
+    sort_buffer = (struct sorthash_t*)calloc(sizeof(struct sorthash_t),1);
+    sort_count = 0;
+
+    list = sorted_file_list(tree);
+    for (place = list; place < list + file_count; place++)
+       shredfile(*place, corehook);
+
+    /* caller is responsible for freeing this */
+    return(sort_buffer);
+}
+
 /* shredtree.c ends here */