Pre-elimination may speed things up significantly.

diff --git a/main.c b/main.c
index d2a7d0801d24042766104a2d358c26369d5f8dee..89e1b4a396693bfa8c99185c8b09a9e7a08a1ff6 100644 (file)
--- a/main.c
+++ b/main.c
@@ -359,9 +359,10 @@ main(int argc, char *argv[])
     extern char        *optarg;        /* set by getopt */
     extern int optind;         /* set by getopt */
 
-    int status, file_only, compile_only;
+    int status, file_only, compile_only, nonuniques;
     struct scf_t       *scf;
     char *dir, *outfile;
+    struct sorthash_t *mp, *np;
 
     compile_only = file_only = 0;
     dir = outfile = NULL;
@@ -565,13 +566,45 @@ main(int argc, char *argv[])
        sort_hashes(sort_buffer, sort_count);
        report_time("Sort done");
 
+       if (debug)
+       {
+           puts("Chunk list before compaction.");
+           dump_array(sort_buffer, sort_count, NULL);
+       }
+
+       /*
+        * To reduce the size of the in-core working set, we do a 
+        * a pre-elimination of duplicates based on hash key alone, in
+        * linear time.  This may leave in the the array hashes that
+        * all point to the same tree.  We'll catch those in a later phase.
+        * The technique: first mark...
+        */
+       if (!HASHCMP(sort_buffer, sort_buffer+1))
+           sort_buffer[0].hash.start = UNIQUE_FLAG;
+       for (np = sort_buffer+1; np < sort_buffer + sort_count-1; np++)
+           if (!HASHCMP(np, np-1) && !HASHCMP(np, np+1))
+               np->hash.start = UNIQUE_FLAG;
+       if (!HASHCMP(sort_buffer+sort_count-2, sort_buffer+sort_count-1))
+           sort_buffer[sort_count-1].hash.start = UNIQUE_FLAG;
+       /* ...then sweep. */
+       nonuniques = 0;
+       for (mp = np = sort_buffer+1; np < sort_buffer + sort_count-1; np++)
+           if (np->hash.start != UNIQUE_FLAG && mp < np)
+           {
+               *mp = *np;
+               nonuniques++;
+           }
+       sort_buffer = (struct sorthash_t *)realloc(sort_buffer, 
+                              sizeof(struct sorthash_t)* nonuniques);
+       report_time("Compaction reduced %d entries to %d", 
+                   sort_count, nonuniques);
+       sort_count = nonuniques;
+
        if (debug)
        {
            puts("Chunk list before reduction.");
            dump_array(sort_buffer, sort_count, NULL);
        }
-#ifdef DEBUG
-#endif /* DEBUG */
 
        emit_report(sort_buffer, sort_count);
     }

diff --git a/shred.h b/shred.h
index dd52ba781699452a6c88940ef1e74d87d049261f..4c18b6ea390f96d294357410e2d255c8aeb9d177 100644 (file)
--- a/shred.h
+++ b/shred.h
@@ -9,6 +9,7 @@
 typedef u_int16_t      linenum_t;
 #define TONET          htons
 #define FROMNET                ntohs
+#define UNIQUE_FLAG    (linenum_t)-1
 
 /*
  * The code depends on the assumption that this structure has