From: Eric S. Raymond <esr@thyrsus.com>
Date: Thu, 11 Sep 2003 19:06:52 +0000 (+0000)
Subject: Pre-elimination may speed things up significantly.
X-Git-Tag: 1.3~20
X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=18ad7e8dedd1b436af9c8c22468a410b744750aa;p=comparator.git

Pre-elimination may speed things up significantly.
---

diff --git a/main.c b/main.c
index d2a7d08..89e1b4a 100644
--- a/main.c
+++ b/main.c
@@ -359,9 +359,10 @@ main(int argc, char *argv[])
     extern char	*optarg;	/* set by getopt */
     extern int	optind;		/* set by getopt */
 
-    int status, file_only, compile_only;
+    int status, file_only, compile_only, nonuniques;
     struct scf_t	*scf;
     char *dir, *outfile;
+    struct sorthash_t *mp, *np;
 
     compile_only = file_only = 0;
     dir = outfile = NULL;
@@ -565,13 +566,45 @@ main(int argc, char *argv[])
 	sort_hashes(sort_buffer, sort_count);
 	report_time("Sort done");
 
+	if (debug)
+	{
+	    puts("Chunk list before compaction.");
+	    dump_array(sort_buffer, sort_count, NULL);
+	}
+
+	/*
+	 * To reduce the size of the in-core working set, we do a 
+	 * a pre-elimination of duplicates based on hash key alone, in
+	 * linear time.  This may leave in the the array hashes that
+	 * all point to the same tree.  We'll catch those in a later phase.
+	 * The technique: first mark...
+	 */
+	if (!HASHCMP(sort_buffer, sort_buffer+1))
+	    sort_buffer[0].hash.start = UNIQUE_FLAG;
+	for (np = sort_buffer+1; np < sort_buffer + sort_count-1; np++)
+	    if (!HASHCMP(np, np-1) && !HASHCMP(np, np+1))
+		np->hash.start = UNIQUE_FLAG;
+	if (!HASHCMP(sort_buffer+sort_count-2, sort_buffer+sort_count-1))
+	    sort_buffer[sort_count-1].hash.start = UNIQUE_FLAG;
+	/* ...then sweep. */
+	nonuniques = 0;
+	for (mp = np = sort_buffer+1; np < sort_buffer + sort_count-1; np++)
+	    if (np->hash.start != UNIQUE_FLAG && mp < np)
+	    {
+		*mp = *np;
+		nonuniques++;
+	    }
+	sort_buffer = (struct sorthash_t *)realloc(sort_buffer, 
+			       sizeof(struct sorthash_t)* nonuniques);
+	report_time("Compaction reduced %d entries to %d", 
+		    sort_count, nonuniques);
+	sort_count = nonuniques;
+
 	if (debug)
 	{
 	    puts("Chunk list before reduction.");
 	    dump_array(sort_buffer, sort_count, NULL);
 	}
-#ifdef DEBUG
-#endif /* DEBUG */
 
 	emit_report(sort_buffer, sort_count);
     }
diff --git a/shred.h b/shred.h
index dd52ba7..4c18b6e 100644
--- a/shred.h
+++ b/shred.h
@@ -9,6 +9,7 @@
 typedef u_int16_t	linenum_t;
 #define TONET		htons
 #define FROMNET		ntohs
+#define UNIQUE_FLAG	(linenum_t)-1
 
 /*
  * The code depends on the assumption that this structure has