New-style normalizer option.

diff --git a/Makefile b/Makefile
index 4e2b34acf04bb0ca040a0fe4a0b1a380dc53d95c..0363df42fa47947f9bc2aa004fac57d9d81e0313 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -51,23 +51,24 @@ comparator.html: comparator.xml
 scf-standard.html: scf-standard.xml
        xmlto html-nochunks scf-standard.xml
 
+OPTS="-N line-oriented, remove-braces, remove-whitespace"
 makeregress:
        @for n in 1 2 3; do \
-           comparator -C -d test test$${n}-a test$${n}-b >test/out$${n}.good;\
+           comparator $(OPTS) -d test test$${n}-a test$${n}-b >test/out$${n}.good;\
        done
 
 regress:
        @for n in 1 2 3; do \
-           comparator -C -d test test$${n}-a test$${n}-b >test/out$${n}.log;\
+           comparator $(OPTS) -d test test$${n}-a test$${n}-b >test/out$${n}.log;\
            if diff -c test/out$${n}.good test/out$${n}.log; \
            then \
                echo "Test $${n} from trees passed."; \
            else \
                echo "Test $${n} from trees failed."; \
            fi; \
-           comparator -C -d test -c test$${n}-a >test$${n}-a.scf; \
-           comparator -C -d test -c test$${n}-b >test$${n}-b.scf; \
-           comparator -C test$${n}-a.scf test$${n}-b.scf >test/out$${n}.log;\
+           comparator $(OPTS) -d test -c test$${n}-a >test$${n}-a.scf; \
+           comparator $(OPTS) -d test -c test$${n}-b >test$${n}-b.scf; \
+           comparator $(OPTS) test$${n}-a.scf test$${n}-b.scf >test/out$${n}.log;\
            rm test$${n}-a.scf test$${n}-b.scf; \
            if diff -u test/out$${n}.good test/out$${n}.log; \
            then \

diff --git a/comparator.xml b/comparator.xml
index 3ca0d4f2bfb1d085e8c9b2e2c69db89399e80f0c..d507233ee36431465c98591d0cb1bb051f9b8b29 100644 (file)
--- a/comparator.xml
+++ b/comparator.xml
@@ -21,6 +21,7 @@
   <arg choice='opt'>-d <replaceable>dir</replaceable></arg>
   <arg choice='opt'>-m <replaceable>minsize</replaceable></arg>
   <arg choice='opt'>-n</arg>
+  <arg choice='opt'>-N <replaceable>normalization-spec</replaceable></arg>
   <arg choice='opt'>-o <replaceable>file</replaceable></arg>
   <arg choice='opt'>-r</arg>
   <arg choice='opt'>-s <replaceable>shredsize</replaceable></arg>
@@ -119,27 +120,11 @@ controls how likely common segments are to not get recognized because
 of unshared things near them, while minimum span size defines the
 smallest features we want to see in the output report.</para>
 
-<para>The <option>-w</option> causes all whitespace in the file
-(including blank lines) to be ignored for comparison purposes (line
-numbers in the output report will nevertheless be correct).</para>
-
 <para>Normally, <application>comparator</application> performs 
 insignificance filtering before emitting a span into the common-segment
 report. The <option>-n</option> suppresses this, emitting all common
 spans. See the discussion of significance filtering below.</para>
 
-<para>The <option>-C</option> option implies <option>-w</option> and also
-applies C-language-specific normalization; { and } characters are
-removed.  This is recommended for comparing C code; it prevents the
-comparison from being fooled by differences in indent style.</para>
-
-<para>The <option>-r</option> option removes comments.  This is only
-effective in conjunction with <option>-C</option>. In C files, ll // comments,
-but only "winged" /* comments with the start and end of comment on the
-same line are ignored; also, block /* comments are retained, but the
-beginning /* and ending */ are ignored, In other files, all #
-comments are removed.</para>
-
 <para>The option <option>-v</option> enables progress and timing
 messages to standard error.</para>
 
@@ -156,6 +141,37 @@ percentage of printable characters near the front of the file is above
 90%.</para>
 </refsect2>
 
+<refsect2><title>Normalization specifications</title>
+
+<para>The <option>-N</option> option passes in its argument as a
+normalization specification.  A normalization specification consists
+of a comma-separated list of tokens.  The first token names a
+normalizer, e.g., a set of rules for transforming code into a
+canonical form.  Subsequent tokens (if present) are options to the
+normalizer.</para>
+
+<refsect3><title>The line-oriented normalizer</title>
+
+<para>The default normalizer is named by the leading token 
+<option>line-oriented</option>.  It has the following options:</para>
+
+<para>The <option>remove-whitespace</option> causes all whitespace in the file
+(including blank lines) to be ignored for comparison purposes (line
+numbers in the output report will nevertheless be correct).</para>
+
+<para>The <option>remove-braces</option> option strips { and }
+characters.  This is recommended for comparing C code; it prevents the
+comparison from being fooled by differences in indent style.</para>
+
+<para>The <option>remove_comments</option> option removes comments in
+C files, All // comments are removed, but only "winged" /* comments
+with the start and end of comment on the same line are ignored; also,
+block /* comments are retained, but the beginning /* and ending */ are
+ignored, In other files, all # comments are removed.</para>
+
+</refsect3>
+</refsect2>
+
 <refsect2><title>Significance filtering</title>
 
 <para>Normally, <application>comparator</application> tries to

diff --git a/linebyline.c b/linebyline.c
index 5d8ffea7928960f6fd44430aad9ca927f34ad06d..86134f4f6235e46a3b30bedc0996fbfb2152cf9c 100644 (file)
--- a/linebyline.c
+++ b/linebyline.c
@@ -52,10 +52,11 @@ static regex_t shell_regexps[sizeof(shell_patterns)/sizeof(*shell_patterns)];
 
 static linenum_t       linecount;
 
-void analyzer_init(unsigned int flags)
+int analyzer_init(unsigned char *buf)
 /* initialize line filtering */
 {
     int i;
+    char       *cp;
 
     for (i = 0; i < sizeof(c_patterns)/sizeof(*c_patterns); i++)
        if (regcomp(c_regexps+i, c_patterns[i], REG_EXTENDED))
@@ -72,12 +73,21 @@ void analyzer_init(unsigned int flags)
            exit(1);
        }
 
-    if (flags & CAPC_FLAG)
-       remove_braces = remove_whitespace = 1;
-    if (flags & R_FLAG)
-       remove_comments = 1;
-    if (flags & W_FLAG)
-       remove_whitespace = 1;
+    cp = strtok(buf, ", ");
+    if (strcmp(cp, "line-oriented"))
+       return(1);
+    while (cp = strtok(NULL, ", "))
+    {
+       if (strcmp(cp, "remove-whitespace") == 0)
+           remove_whitespace = 1;
+       else if (strcmp(cp, "remove-comments") == 0)
+           remove_comments = 1;
+       else if (strcmp(cp, "remove-braces") == 0)
+           remove_braces = 1;
+       else
+           return(-1);
+    }
+    return(0);
 }
 
 static unsigned char   active = 0;
@@ -109,7 +119,7 @@ static int normalize(char *buf)
 {
     if (remove_comments)
     {
-       if (remove_braces)      /* remove C comments */
+       if (active & C_CODE)    /* remove C comments */
        {
            char        *ss = strstr(buf, "//");
 
@@ -289,7 +299,7 @@ struct analyzer_t linebyline =
     mode: analyzer_mode,
     get:  analyzer_get,
     free: analyzer_free,
-    dump: analyzer_dump,
+    dumpopt: analyzer_dump,
 };
 
 #ifdef TEST

diff --git a/main.c b/main.c
index b03aae50bf26ab968c9c861572c7ad42b383a3b5..5de5e77c2b3e1a2b24bdc775255682d35e32fd75 100644 (file)
--- a/main.c
+++ b/main.c
@@ -118,7 +118,7 @@ static void write_scf(const char *tree, FILE *ofp)
     fputs("#SCF-A 2.0\n", ofp);
     fputs("Generator-Program: comparator 1.0\n", ofp);
     fputs("Hash-Method: " HASHMETHOD "\n", ofp);
-    linebyline.dump(buf);
+    linebyline.dumpopt(buf);
     fprintf(ofp, "Normalization: %s\n", buf);
     fprintf(ofp, "Root: %s\n", tree);
     fprintf(ofp, "Shred-Size: %d\n", shredsize);
@@ -330,7 +330,7 @@ static void init_scf(char *file, struct scf_t *scf, const int readfile)
        char    buf[BUFSIZ];
 
        scf->hash_method = HASHMETHOD;
-       linebyline.dump(buf);
+       linebyline.dumpopt(buf);
        scf->normalization = strdup(buf);
        scf->shred_size = shredsize;
        scf->generator_program = "comparator " VERSION;
@@ -416,15 +416,12 @@ static void usage(void)
 {
     fprintf(stderr,"usage: comparator [-c] [-C] [-d dir ] [-m minsize] [-n] [-r] [-o file] [-s shredsize] [-v] [-w] [-x] path...\n");
     fprintf(stderr,"  -c      = generate SCF files\n");
-    fprintf(stderr,"  -C      = apply C normalizations\n");
     fprintf(stderr,"  -d dir  = change directory before digesting.\n");
     fprintf(stderr,"  -m size = set minimum size of span to be output.\n");
     fprintf(stderr,"  -n      = suppress significance filtering.\n");
     fprintf(stderr,"  -o file = write to the specified file.\n");
-    fprintf(stderr,"  -r      = renove comments\n");
     fprintf(stderr,"  -s size = set shred size (default %d)\n", shredsize);
     fprintf(stderr,"  -v      = enable progress messages on stderr.\n");
-    fprintf(stderr,"  -w      = remove whitespace.\n");
     fprintf(stderr,"  -x      = debug, display chunks in output.\n");
     fprintf(stderr,"This is comparator version " VERSION ".\n");
     exit(0);
@@ -436,13 +433,13 @@ main(int argc, char *argv[])
     extern char        *optarg;        /* set by getopt */
     extern int optind;         /* set by getopt */
 
-    int status, file_only, compile_only, argcount, mergecount, flags;
+    int status, file_only, compile_only, argcount, mergecount;
     struct scf_t       *scf;
-    char *dir, *outfile;
+    char *dir, *outfile, *normalization = "line_oriented";
 
-    compile_only = file_only = nofilter = flags = 0;
+    compile_only = file_only = nofilter = 0;
     dir = outfile = NULL;
-    while ((status = getopt(argc, argv, "cCd:hm:no:rs:vwx")) != EOF)
+    while ((status = getopt(argc, argv, "cd:hm:nN:o:rs:vx")) != EOF)
     {
        switch (status)
        {
@@ -450,10 +447,6 @@ main(int argc, char *argv[])
            compile_only = 1;
            break;
 
-       case 'C':
-           flags |= CAPC_FLAG;
-           break;
-
        case 'd':
            dir = optarg;
            break;
@@ -466,12 +459,12 @@ main(int argc, char *argv[])
            nofilter = 1;
            break;
 
-       case 'o':
-           outfile = optarg;
+       case 'N':
+           normalization = strdup(optarg);
            break;
 
-       case 'r':
-           flags |= R_FLAG;
+       case 'o':
+           outfile = optarg;
            break;
 
        case 's':
@@ -482,10 +475,6 @@ main(int argc, char *argv[])
            verbose = 1;
            break;
 
-       case 'w':
-           flags |= W_FLAG;
-           break;
-
        case 'x':
            debug = 1;
            break;
@@ -506,7 +495,16 @@ main(int argc, char *argv[])
        usage();
 
     report_time(NULL);
-    linebyline.init(flags);
+
+    /*
+     * Check each normalizer to see if it fires.
+     */
+    status = linebyline.init(normalization);
+    if (status != 0)
+    {
+       fprintf(stderr, "comparator: bad analyzer option\n");
+       exit(1);
+    }
 
     /* special case if user gave exactly one tree */
     if (!compile_only && argcount == 1)

diff --git a/makefile b/makefile
index 0a92d4e6304c90544e3d5adcf6f85de0cbc0ab25..485a089dc34a9de8017dcd5c1ea8798cfbc39949 100644 (file)
--- a/makefile
+++ b/makefile
@@ -7,12 +7,13 @@ include Makefile
 UNIX = /home/esr/src/unix
 SRC1  = sys3
 SRC2  = 32v
+OPTS="-N line-oriented, remove-braces, remove-whitespace"
 
 # Test that making several hash files at one go doesn't mess up. 
 regress2:
-       comparator -C -d $(UNIX) $(SRC1) | dumpscf >test1-old.dump
-       comparator -C -d $(UNIX) $(SRC2) | dumpscf >test2-old.dump
-       comparator -C -c -d $(UNIX) $(SRC1) $(SRC2)
+       comparator $(OPTS) -d $(UNIX) $(SRC1) | dumpscf >test1-old.dump
+       comparator $(OPTS) -d $(UNIX) $(SRC2) | dumpscf >test2-old.dump
+       comparator $(OPTS) -c -d $(UNIX) $(SRC1) $(SRC2)
        dumpscf <$(SRC1).scf >test1.dump
        dumpscf <$(SRC2).scf >test2.dump
        diff -u test1-old.dump test1.dump
@@ -20,11 +21,11 @@ regress2:
 
 # Test that making a report from SCF files works the same as from the trees
 regress3:
-       comparator -C -d $(UNIX) $(SRC1) $(SRC2) >test1.dump
-       comparator -C -c -d $(UNIX) $(SRC1) $(SRC2)
+       comparator $(OPTS) -d $(UNIX) $(SRC1) $(SRC2) >test1.dump
+       comparator $(OPTS) -c -d $(UNIX) $(SRC1) $(SRC2)
        comparator $(SRC1).scf $(SRC2).scf >test2.dump
        diff -u test1.dump test2.dump
 
 # Test SVr4 against Linux
 sco.ovl:
-       time comparator -v -C -d $(UNIX) src5r4 linux-2.4.19 >sco.ovl
+       time comparator -v $(OPTS) -d $(UNIX) src5r4 linux-2.4.19 >sco.ovl

diff --git a/shred.h b/shred.h
index df92cccf28021813602af2b244cfe4afa4c239a6..d774e993f06ff3ff481f6b16a488d893a517f6e2 100644 (file)
--- a/shred.h
+++ b/shred.h
@@ -59,15 +59,11 @@ feature_t;
 
 struct analyzer_t      /* structure describing a feature analyzer */
 {
-    unsigned int flags;
-#define CAPC_FLAG      0x01
-#define R_FLAG         0x02
-#define W_FLAG         0x04
-    void (*init)(unsigned int);
+    int (*init)(unsigned char *);
     void (*mode)(int);
     feature_t *(*get)(const struct filehdr_t *, FILE *, linenum_t *);
     void (*free)(const char *);
-    void (*dump)(char *);
+    void (*dumpopt)(char *);
 };
 
 /* mode bits for controlling the analyzer */