Filtering is working. Now inmoprove the statistics.

diff --git a/linebyline.c b/linebyline.c
index 7fc9ad899db7636df834d07b5e114fd03d4f6a12..189c723be721322bf546061fcf16d457835287a6 100644 (file)
--- a/linebyline.c
+++ b/linebyline.c
@@ -83,12 +83,12 @@ int filter_pass(const char *line)
 /* return flags that apply to this line */
 {
     if (active == 0)
-       return(SIGNIFICANT);
+       return(0);
     else
     {
        char    *sp, *tp;
        char    buf[BUFSIZ];
-       int     changed, significant;
+       int     changed, insignificant;
 
        /* change all punctuation to spaces */
        buf[0] = ' ';
@@ -132,10 +132,7 @@ int filter_pass(const char *line)
        } while
            (changed);
 
-
-       significant = !(buf[0] == '\0' || strspn(buf, " ") == strlen(buf));
-
-       return (active | (significant ? SIGNIFICANT : 0));
+       return (buf[0] == '\0' || strspn(buf, " ") == strlen(buf));
     }
 }
 

diff --git a/main.c b/main.c
index 3ca3b8b6216bc8ed14ad95b42be0e59908cc0964..607e57b767a8064deb5882f24595c95c478611d5 100644 (file)
--- a/main.c
+++ b/main.c
@@ -358,8 +358,8 @@ void dump_flags(const int flags, FILE *fp)
 /* dump tokens corresponding to a flag set */
 {
     fputc(' ', fp);
-    if (flags & SIGNIFICANT)
-       fputs(" significant", fp);
+    if (flags & INSIGNIFICANT)
+       fputs(" insignificant", fp);
     if (flags & C_CODE)
        fputs(" C", fp);
     if (flags & SHELL_CODE)

diff --git a/report.c b/report.c
index 5f720d104c93fa9d0a6d6f03ab65c2ea1391c01e..7d2e7e1b81df47dc91f64ad9b7edf80cad066577 100644 (file)
--- a/report.c
+++ b/report.c
@@ -65,7 +65,12 @@ static int merge_ranges(struct sorthash_t *p,
     {
        p[i].hash.start = min(p[i].hash.start, q[i].hash.start);
        p[i].hash.end   = max(p[i].hash.end, q[i].hash.end);
-       p[i].hash.flags |= q[i].hash.flags;
+       /*
+        * The insignificance bit in the merged range should be cleared
+        * if the range being merged in is significant.  This is important;
+        * it means that significance propagates as spabs merge.
+        */
+       p[i].hash.flags &=~ q[i].hash.flags;
        q[i].hash.flags = INTERNAL_FLAG;        /* used only in debug code */
     }
     return(1);
@@ -400,6 +405,15 @@ void emit_report2(void)
 
            if (matchsize >= maxsize)
                maxsize = matchsize;
+
+           /*
+            * This odd-looking bit of logic, deals with an
+            * annoying edge case.  Sometimes we'll get boilerplate C
+            * stuff in both a C and a text file. To cope, propagate
+            * insignificance -- if we know from one context that a
+            * particular span of text is not interesting, declare it
+            * uninteresting everywhere.
+            */
            flags |= rp->hash.flags;
        }
 
@@ -408,7 +422,7 @@ void emit_report2(void)
         * point of doing it this way is to allow significance bits to 
         * propagate through range merges.
         */
-       if (maxsize < minsize || !(nofilter || (flags & SIGNIFICANT)))
+       if (maxsize < minsize || (!nofilter && (flags & INSIGNIFICANT)))
            continue;
 
        for (i=0; i < match->nmatches; i++)

diff --git a/shred.h b/shred.h
index c67d98f7186fe415fd21b9cc324a8c0906c84e8b..be175e419c05cf39d22a7b19c7f946c5fa534439 100644 (file)
--- a/shred.h
+++ b/shred.h
@@ -31,7 +31,7 @@ struct hash_t
     linenum_t          start, end;
     hashval_t          hash;
     flag_t             flags;
-#define SIGNIFICANT    0x40    /* something in this is not pure syntax */
+#define INSIGNIFICANT  0x40    /* pure syntax in  a programming language */
 #define C_CODE         0x01    /* identified as C code */
 #define SHELL_CODE     0x02    /* identified as shell code */
 #define CATEGORIZED    0x03    /* we can significance-test this */

diff --git a/shredtree.c b/shredtree.c
index d7ddb3a4dba6e576949d3f0a480723bbf02616ce..0613169c008bc8c68b8d267607deac38c8a00627 100644 (file)
--- a/shredtree.c
+++ b/shredtree.c
@@ -228,7 +228,7 @@ int shredfile(struct filehdr_t *file,
        /* create new shred */
        display[shredsize-1].line = strdup(buf);
        display[shredsize-1].start = linecount;
-       display[shredsize-1].flags = filter_pass(buf);
+       display[shredsize-1].flags = filter_pass(buf) ? INSIGNIFICANT : 0;
 
        /* flush completed chunk */
        if (accepted >= shredsize)