From: Eric S. Raymond <esr@thyrsus.com>
Date: Thu, 18 Sep 2003 06:12:07 +0000 (+0000)
Subject: Filtering is working.  Now inmoprove the statistics.
X-Git-Tag: 2.0~16
X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=163bb2d46f76ca5543bd49e9dbabb2cc720eddd3;p=comparator.git

Filtering is working.  Now inmoprove the statistics.
---

diff --git a/linebyline.c b/linebyline.c
index 7fc9ad8..189c723 100644
--- a/linebyline.c
+++ b/linebyline.c
@@ -83,12 +83,12 @@ int filter_pass(const char *line)
 /* return flags that apply to this line */
 {
     if (active == 0)
-	return(SIGNIFICANT);
+	return(0);
     else
     {
 	char	*sp, *tp;
 	char	buf[BUFSIZ];
-	int	changed, significant;
+	int	changed, insignificant;
 
 	/* change all punctuation to spaces */
 	buf[0] = ' ';
@@ -132,10 +132,7 @@ int filter_pass(const char *line)
 	} while
 	    (changed);
 
-
-	significant = !(buf[0] == '\0' || strspn(buf, " ") == strlen(buf));
-
-	return (active | (significant ? SIGNIFICANT : 0));
+	return (buf[0] == '\0' || strspn(buf, " ") == strlen(buf));
     }
 }
 
diff --git a/main.c b/main.c
index 3ca3b8b..607e57b 100644
--- a/main.c
+++ b/main.c
@@ -358,8 +358,8 @@ void dump_flags(const int flags, FILE *fp)
 /* dump tokens corresponding to a flag set */
 {
     fputc(' ', fp);
-    if (flags & SIGNIFICANT)
-	fputs(" significant", fp);
+    if (flags & INSIGNIFICANT)
+	fputs(" insignificant", fp);
     if (flags & C_CODE)
 	fputs(" C", fp);
     if (flags & SHELL_CODE)
diff --git a/report.c b/report.c
index 5f720d1..7d2e7e1 100644
--- a/report.c
+++ b/report.c
@@ -65,7 +65,12 @@ static int merge_ranges(struct sorthash_t *p,
     {
 	p[i].hash.start = min(p[i].hash.start, q[i].hash.start);
 	p[i].hash.end   = max(p[i].hash.end, q[i].hash.end);
-	p[i].hash.flags |= q[i].hash.flags;
+	/*
+	 * The insignificance bit in the merged range should be cleared
+	 * if the range being merged in is significant.  This is important;
+	 * it means that significance propagates as spabs merge.
+	 */
+	p[i].hash.flags &=~ q[i].hash.flags;
 	q[i].hash.flags = INTERNAL_FLAG;	/* used only in debug code */
     }
     return(1);
@@ -400,6 +405,15 @@ void emit_report2(void)
 
 	    if (matchsize >= maxsize)
 		maxsize = matchsize;
+
+	    /*
+	     * This odd-looking bit of logic, deals with an
+	     * annoying edge case.  Sometimes we'll get boilerplate C
+	     * stuff in both a C and a text file. To cope, propagate
+	     * insignificance -- if we know from one context that a
+	     * particular span of text is not interesting, declare it
+	     * uninteresting everywhere.
+	     */
 	    flags |= rp->hash.flags;
 	}
 
@@ -408,7 +422,7 @@ void emit_report2(void)
 	 * point of doing it this way is to allow significance bits to 
 	 * propagate through range merges.
 	 */
-	if (maxsize < minsize || !(nofilter || (flags & SIGNIFICANT)))
+	if (maxsize < minsize || (!nofilter && (flags & INSIGNIFICANT)))
 	    continue;
 
 	for (i=0; i < match->nmatches; i++)
diff --git a/shred.h b/shred.h
index c67d98f..be175e4 100644
--- a/shred.h
+++ b/shred.h
@@ -31,7 +31,7 @@ struct hash_t
     linenum_t   	start, end;
     hashval_t		hash;
     flag_t		flags;
-#define SIGNIFICANT	0x40	/* something in this is not pure syntax */
+#define INSIGNIFICANT	0x40	/* pure syntax in  a programming language */
 #define C_CODE		0x01	/* identified as C code */
 #define SHELL_CODE	0x02	/* identified as shell code */
 #define CATEGORIZED	0x03	/* we can significance-test this */
diff --git a/shredtree.c b/shredtree.c
index d7ddb3a..0613169 100644
--- a/shredtree.c
+++ b/shredtree.c
@@ -228,7 +228,7 @@ int shredfile(struct filehdr_t *file,
 	/* create new shred */
 	display[shredsize-1].line = strdup(buf);
 	display[shredsize-1].start = linecount;
-	display[shredsize-1].flags = filter_pass(buf);
+	display[shredsize-1].flags = filter_pass(buf) ? INSIGNIFICANT : 0;
 
 	/* flush completed chunk */
 	if (accepted >= shredsize)