From: Eric S. Raymond <esr@thyrsus.com>
Date: Thu, 11 Sep 2003 20:40:02 +0000 (+0000)
Subject: Better comment.
X-Git-Tag: 1.3~12
X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=6b9544884ea1cd009f96420ae7a8395a4593dcfb;p=comparator.git

Better comment.
---

diff --git a/report.c b/report.c
index a09d183..5606d15 100644
--- a/report.c
+++ b/report.c
@@ -107,30 +107,40 @@ struct match_t *reduce_matches(struct sorthash_t *obarray, int hashcount)
      unsigned int retry, nonuniques, progress;
      struct sorthash_t *mp, *np;
 
-    /*
-     * To reduce the size of the in-core working set, we do a 
-     * a pre-elimination of duplicates based on hash key alone, in
-     * linear time.  This may leave in the the array hashes that
-     * all point to the same tree.  We'll catch those in a later phase.
-     * The technique: first mark...
-     */
-    if (!HASHCMP(obarray, obarray+1))
-	obarray[0].hash.start = UNIQUE_FLAG;
-    for (np = obarray+1; np < obarray + hashcount-1; np++)
-	if (!HASHCMP(np, np-1) && !HASHCMP(np, np+1))
-	    np->hash.start = UNIQUE_FLAG;
-    if (!HASHCMP(obarray+hashcount-2, obarray+hashcount-1))
-	obarray[hashcount-1].hash.start = UNIQUE_FLAG;
-    /* ...then sweep. */
-    for (mp = np = obarray; np < obarray + hashcount; np++)
-	if (np->hash.start != UNIQUE_FLAG && mp < np)
-	    *mp++ = *np;
-    /* now we get to reduce the memory footprint */
-    report_time("Compaction reduced %d shreds to %d", 
-		hashcount, mp - obarray);
-    hashcount = mp - obarray;
-    obarray = (struct sorthash_t *)realloc(obarray, 
-			   sizeof(struct sorthash_t)* hashcount);
+     /*
+      * To reduce the size of the in-core working set, we do a a
+      * pre-elimination of duplicates based on hash key alone, in
+      * linear time.  This may leave in the array hashes that all
+      * point to the same tree.  We'll discard those in the next
+      * phase.  This costs less time than one might think; without it,
+      * the clique detector in the next phase would have to do an
+      * extra HASHCMP per array slot to detect clique boundaries.  Net
+      * cost of this optimization is thus *one* HASHCMP per entry.
+      * The benefit is that it kicks lots of shreds out, reducing the
+      * total working set at the time we build match lists.  The idea
+      * here is to avoid swapping, because typical data sets are so
+      * large that handling them can easily kick the VM into
+      * thrashing.
+      *
+      * The technique: first mark...
+      */
+     if (!HASHCMP(obarray, obarray+1))
+	 obarray[0].hash.start = UNIQUE_FLAG;
+     for (np = obarray+1; np < obarray + hashcount-1; np++)
+	 if (!HASHCMP(np, np-1) && !HASHCMP(np, np+1))
+	     np->hash.start = UNIQUE_FLAG;
+     if (!HASHCMP(obarray+hashcount-2, obarray+hashcount-1))
+	 obarray[hashcount-1].hash.start = UNIQUE_FLAG;
+     /* ...then sweep. */
+     for (mp = np = obarray; np < obarray + hashcount; np++)
+	 if (np->hash.start != UNIQUE_FLAG && mp < np)
+	     *mp++ = *np;
+     /* now we get to reduce the memory footprint */
+     report_time("Compaction reduced %d shreds to %d", 
+		 hashcount, mp - obarray);
+     hashcount = mp - obarray;
+     obarray = (struct sorthash_t *)realloc(obarray, 
+			    sizeof(struct sorthash_t)* hashcount);
 
      /* build list of hashes with more than one range associated with */
      nonuniques = progress = 0;