From: Eric S. Raymond <esr@thyrsus.com>
Date: Thu, 18 Sep 2003 05:13:23 +0000 (+0000)
Subject: Improve filtering.
X-Git-Tag: 2.0~17
X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=50ea48aa52ab6e9162ae672e3282e1038f5691d7;p=comparator.git

Improve filtering.
---

diff --git a/linebyline.c b/linebyline.c
index 0547953..7fc9ad8 100644
--- a/linebyline.c
+++ b/linebyline.c
@@ -7,23 +7,23 @@
 
 static char *c_patterns[] = {
     /* Idioms that don't convey any meaning in isolation */
-    "return [a-z]+", "return -?[01]+",
-    "goto +[a-z]+", "exit *\\([01]\\)",
+    "return [a-z]+", "return [01]+",
+    "goto +[a-z]+", "exit *[01]",
     /* Pragmas */
-    "/\\* *ARGSUSED *\\*/",
-    "/\\* *NOTREACHED *\\*/", 
-    "/\\* *FALL *THRO?UG?H? *\\*/",
+    " ARGSUSED ",
+    " NOTREACHED ", 
+    " FALL *THRO?UG?H? ",
     /* Bare C keywords */
     " auto ", " break ",  " case "," continue ", " default ", " do ", " else ",
     " enum ", " if ", " goto "," return ", " switch ", " while ", " int ",
     " long ", " short ", " static ", " struct ", " typedef ", " union ", 
     " void ",
-    /* Preprocessor constructs (# has already been stripped)*/
-    "^ *define","^ *endif","^ *else", "^ *ifdef ","^ *ifndef ",
+    /* Preprocessor constructs (# has already been stripped) */
+    "^ define"," endif"," else", " ifdef "," ifndef ",
     /* Common preprocessor macros, not significant by themselves. */
     " ASSERT ", " EXTERN ", " FALSE ", " NULL "," STATIC ", " TRUE ",
     /* Macro include lines are noise, too. */
-    "^ *include.*","^ *line.*",
+    " include .*"," line .*",
     /* Common error macros. */
     " EFAULT ",
     " EINVAL ",
diff --git a/main.c b/main.c
index 1893626..3ca3b8b 100644
--- a/main.c
+++ b/main.c
@@ -374,19 +374,20 @@ void dump_array(const char *legend,
 
     fputs(legend, stdout);
     for (np = obarray; np < obarray + hashcount; np++)
-    {
-	fprintf(stdout, "%2d: %s %s:%d:%d",
-		np-obarray, 
-		hash_dump(np->hash.hash),
-		np->file->name, np->hash.start, np->hash.end);
-	if (np->hash.flags)
+	if (!(np->hash.flags & INTERNAL_FLAG))
 	{
-	    fputc('\t', stdout);
-	    dump_flags(np->hash.flags, stdout);
-	    fprintf(stdout, " (0x%02x)", np->hash.flags);
+	    fprintf(stdout, "%2d: %s %s:%d:%d",
+		    np-obarray, 
+		    hash_dump(np->hash.hash),
+		    np->file->name, np->hash.start, np->hash.end);
+	    if (np->hash.flags)
+	    {
+		fputc('\t', stdout);
+		dump_flags(np->hash.flags, stdout);
+		fprintf(stdout, " (0x%02x)", np->hash.flags);
+	    }
+	    fputc('\n', stdout);
 	}
-	fputc('\n', stdout);
-    }
 }
 
 void report_time(char *legend, ...)
diff --git a/report.c b/report.c
index 5f6005d..5f720d1 100644
--- a/report.c
+++ b/report.c
@@ -66,6 +66,7 @@ static int merge_ranges(struct sorthash_t *p,
 	p[i].hash.start = min(p[i].hash.start, q[i].hash.start);
 	p[i].hash.end   = max(p[i].hash.end, q[i].hash.end);
 	p[i].hash.flags |= q[i].hash.flags;
+	q[i].hash.flags = INTERNAL_FLAG;	/* used only in debug code */
     }
     return(1);
 }
@@ -226,15 +227,15 @@ static int compact_matches(struct sorthash_t *obarray, const int hashcount)
       * The technique: first mark...
       */
      if (SORTHASHCMP(obarray, obarray+1))
-	 obarray[0].hash.flags = UNIQUE_FLAG;
+	 obarray[0].hash.flags = INTERNAL_FLAG;
      for (np = obarray+1; np < obarray + hashcount-1; np++)
 	 if (SORTHASHCMP(np, np-1) && SORTHASHCMP(np, np+1))
-	     np->hash.flags = UNIQUE_FLAG;
+	     np->hash.flags = INTERNAL_FLAG;
      if (SORTHASHCMP(obarray+hashcount-2, obarray+hashcount-1))
-	 obarray[hashcount-1].hash.flags = UNIQUE_FLAG;
+	 obarray[hashcount-1].hash.flags = INTERNAL_FLAG;
      /* ...then sweep. */
      for (mp = np = obarray; np < obarray + hashcount; np++)
-	 if (np->hash.flags != UNIQUE_FLAG)
+	 if (np->hash.flags != INTERNAL_FLAG)
 	     *mp++ = *np;
      /* now we get to reduce the memory footprint */
      report_time("Compaction reduced %d shreds to %d", 
@@ -282,7 +283,10 @@ struct match_t *reduce_matches(struct sorthash_t *obarray, int *hashcountp)
 	     if (!sametree(np[i].file->name, np[(i+1) % nmatches].file->name))
 		 heterogenous++;
 	 if (!heterogenous)
+	 {
+	     np[i].hash.flags = INTERNAL_FLAG;	/* used only in debug code */
 	     continue;
+	 }
 
 	 if (debug)
 	 {
@@ -362,6 +366,8 @@ void emit_report1(struct sorthash_t *obarray, int hashcount)
 	dump_array("After removing uniques.\n", obarray, hashcount);
     report_time("%d range groups after removing unique hashes", matchcount);
     mergecount = collapse_ranges(hitlist, matchcount);
+    if (debug)
+	dump_array("After merging ranges.\n", obarray, hashcount);
     report_time("%d range groups after merging", mergecount);
 
     /*
diff --git a/shred.h b/shred.h
index 3270382..c67d98f 100644
--- a/shred.h
+++ b/shred.h
@@ -35,7 +35,7 @@ struct hash_t
 #define C_CODE		0x01	/* identified as C code */
 #define SHELL_CODE	0x02	/* identified as shell code */
 #define CATEGORIZED	0x03	/* we can significance-test this */
-#define UNIQUE_FLAG	0x80	/* internal use only */
+#define INTERNAL_FLAG	0x80	/* internal use only */
 };
 #define SORTHASHCMP(s, t) hash_compare((s)->hash.hash, (t)->hash.hash)
 
diff --git a/shredtree.c b/shredtree.c
index a04537b..d7ddb3a 100644
--- a/shredtree.c
+++ b/shredtree.c
@@ -40,7 +40,7 @@ static int eligible(const char *file)
 #define endswith(suff)	!strcmp(suff, file + strlen(file) - strlen(suff))
     if (endswith(".c") || endswith(".h") || endswith(".html"))
 	return(1);
-    else if (endswith(".o") || endswith("~"))
+    else if (endswith(".o") || endswith("~") || endswith(".bdf"))
 	return(0);
     else if (strstr(file, "CVS") || strstr(file,"RCS") || strstr(file,"SCCS"))
 	return(0);