From afa8c3eec0ee4aad286249d7e59d718a0136d0eb Mon Sep 17 00:00:00 2001 From: "Eric S. Raymond" Date: Sat, 27 Sep 2003 08:08:57 +0000 Subject: [PATCH] New-style normalizer option. --- Makefile | 11 ++++++----- comparator.xml | 48 ++++++++++++++++++++++++++++++++---------------- linebyline.c | 28 +++++++++++++++++++--------- main.c | 42 ++++++++++++++++++++---------------------- makefile | 13 +++++++------ shred.h | 8 ++------ 6 files changed, 86 insertions(+), 64 deletions(-) diff --git a/Makefile b/Makefile index 4e2b34a..0363df4 100644 --- a/Makefile +++ b/Makefile @@ -51,23 +51,24 @@ comparator.html: comparator.xml scf-standard.html: scf-standard.xml xmlto html-nochunks scf-standard.xml +OPTS="-N line-oriented, remove-braces, remove-whitespace" makeregress: @for n in 1 2 3; do \ - comparator -C -d test test$${n}-a test$${n}-b >test/out$${n}.good;\ + comparator $(OPTS) -d test test$${n}-a test$${n}-b >test/out$${n}.good;\ done regress: @for n in 1 2 3; do \ - comparator -C -d test test$${n}-a test$${n}-b >test/out$${n}.log;\ + comparator $(OPTS) -d test test$${n}-a test$${n}-b >test/out$${n}.log;\ if diff -c test/out$${n}.good test/out$${n}.log; \ then \ echo "Test $${n} from trees passed."; \ else \ echo "Test $${n} from trees failed."; \ fi; \ - comparator -C -d test -c test$${n}-a >test$${n}-a.scf; \ - comparator -C -d test -c test$${n}-b >test$${n}-b.scf; \ - comparator -C test$${n}-a.scf test$${n}-b.scf >test/out$${n}.log;\ + comparator $(OPTS) -d test -c test$${n}-a >test$${n}-a.scf; \ + comparator $(OPTS) -d test -c test$${n}-b >test$${n}-b.scf; \ + comparator $(OPTS) test$${n}-a.scf test$${n}-b.scf >test/out$${n}.log;\ rm test$${n}-a.scf test$${n}-b.scf; \ if diff -u test/out$${n}.good test/out$${n}.log; \ then \ diff --git a/comparator.xml b/comparator.xml index 3ca0d4f..d507233 100644 --- a/comparator.xml +++ b/comparator.xml @@ -21,6 +21,7 @@ -d dir -m minsize -n + -N normalization-spec -o file -r -s shredsize @@ -119,27 +120,11 @@ controls how likely common segments are to not get recognized because of unshared things near them, while minimum span size defines the smallest features we want to see in the output report. -The causes all whitespace in the file -(including blank lines) to be ignored for comparison purposes (line -numbers in the output report will nevertheless be correct). - Normally, comparator performs insignificance filtering before emitting a span into the common-segment report. The suppresses this, emitting all common spans. See the discussion of significance filtering below. -The option implies and also -applies C-language-specific normalization; { and } characters are -removed. This is recommended for comparing C code; it prevents the -comparison from being fooled by differences in indent style. - -The option removes comments. This is only -effective in conjunction with . In C files, ll // comments, -but only "winged" /* comments with the start and end of comment on the -same line are ignored; also, block /* comments are retained, but the -beginning /* and ending */ are ignored, In other files, all # -comments are removed. - The option enables progress and timing messages to standard error. @@ -156,6 +141,37 @@ percentage of printable characters near the front of the file is above 90%. +Normalization specifications + +The option passes in its argument as a +normalization specification. A normalization specification consists +of a comma-separated list of tokens. The first token names a +normalizer, e.g., a set of rules for transforming code into a +canonical form. Subsequent tokens (if present) are options to the +normalizer. + +The line-oriented normalizer + +The default normalizer is named by the leading token +. It has the following options: + +The causes all whitespace in the file +(including blank lines) to be ignored for comparison purposes (line +numbers in the output report will nevertheless be correct). + +The option strips { and } +characters. This is recommended for comparing C code; it prevents the +comparison from being fooled by differences in indent style. + +The option removes comments in +C files, All // comments are removed, but only "winged" /* comments +with the start and end of comment on the same line are ignored; also, +block /* comments are retained, but the beginning /* and ending */ are +ignored, In other files, all # comments are removed. + + + + Significance filtering Normally, comparator tries to diff --git a/linebyline.c b/linebyline.c index 5d8ffea..86134f4 100644 --- a/linebyline.c +++ b/linebyline.c @@ -52,10 +52,11 @@ static regex_t shell_regexps[sizeof(shell_patterns)/sizeof(*shell_patterns)]; static linenum_t linecount; -void analyzer_init(unsigned int flags) +int analyzer_init(unsigned char *buf) /* initialize line filtering */ { int i; + char *cp; for (i = 0; i < sizeof(c_patterns)/sizeof(*c_patterns); i++) if (regcomp(c_regexps+i, c_patterns[i], REG_EXTENDED)) @@ -72,12 +73,21 @@ void analyzer_init(unsigned int flags) exit(1); } - if (flags & CAPC_FLAG) - remove_braces = remove_whitespace = 1; - if (flags & R_FLAG) - remove_comments = 1; - if (flags & W_FLAG) - remove_whitespace = 1; + cp = strtok(buf, ", "); + if (strcmp(cp, "line-oriented")) + return(1); + while (cp = strtok(NULL, ", ")) + { + if (strcmp(cp, "remove-whitespace") == 0) + remove_whitespace = 1; + else if (strcmp(cp, "remove-comments") == 0) + remove_comments = 1; + else if (strcmp(cp, "remove-braces") == 0) + remove_braces = 1; + else + return(-1); + } + return(0); } static unsigned char active = 0; @@ -109,7 +119,7 @@ static int normalize(char *buf) { if (remove_comments) { - if (remove_braces) /* remove C comments */ + if (active & C_CODE) /* remove C comments */ { char *ss = strstr(buf, "//"); @@ -289,7 +299,7 @@ struct analyzer_t linebyline = mode: analyzer_mode, get: analyzer_get, free: analyzer_free, - dump: analyzer_dump, + dumpopt: analyzer_dump, }; #ifdef TEST diff --git a/main.c b/main.c index b03aae5..5de5e77 100644 --- a/main.c +++ b/main.c @@ -118,7 +118,7 @@ static void write_scf(const char *tree, FILE *ofp) fputs("#SCF-A 2.0\n", ofp); fputs("Generator-Program: comparator 1.0\n", ofp); fputs("Hash-Method: " HASHMETHOD "\n", ofp); - linebyline.dump(buf); + linebyline.dumpopt(buf); fprintf(ofp, "Normalization: %s\n", buf); fprintf(ofp, "Root: %s\n", tree); fprintf(ofp, "Shred-Size: %d\n", shredsize); @@ -330,7 +330,7 @@ static void init_scf(char *file, struct scf_t *scf, const int readfile) char buf[BUFSIZ]; scf->hash_method = HASHMETHOD; - linebyline.dump(buf); + linebyline.dumpopt(buf); scf->normalization = strdup(buf); scf->shred_size = shredsize; scf->generator_program = "comparator " VERSION; @@ -416,15 +416,12 @@ static void usage(void) { fprintf(stderr,"usage: comparator [-c] [-C] [-d dir ] [-m minsize] [-n] [-r] [-o file] [-s shredsize] [-v] [-w] [-x] path...\n"); fprintf(stderr," -c = generate SCF files\n"); - fprintf(stderr," -C = apply C normalizations\n"); fprintf(stderr," -d dir = change directory before digesting.\n"); fprintf(stderr," -m size = set minimum size of span to be output.\n"); fprintf(stderr," -n = suppress significance filtering.\n"); fprintf(stderr," -o file = write to the specified file.\n"); - fprintf(stderr," -r = renove comments\n"); fprintf(stderr," -s size = set shred size (default %d)\n", shredsize); fprintf(stderr," -v = enable progress messages on stderr.\n"); - fprintf(stderr," -w = remove whitespace.\n"); fprintf(stderr," -x = debug, display chunks in output.\n"); fprintf(stderr,"This is comparator version " VERSION ".\n"); exit(0); @@ -436,13 +433,13 @@ main(int argc, char *argv[]) extern char *optarg; /* set by getopt */ extern int optind; /* set by getopt */ - int status, file_only, compile_only, argcount, mergecount, flags; + int status, file_only, compile_only, argcount, mergecount; struct scf_t *scf; - char *dir, *outfile; + char *dir, *outfile, *normalization = "line_oriented"; - compile_only = file_only = nofilter = flags = 0; + compile_only = file_only = nofilter = 0; dir = outfile = NULL; - while ((status = getopt(argc, argv, "cCd:hm:no:rs:vwx")) != EOF) + while ((status = getopt(argc, argv, "cd:hm:nN:o:rs:vx")) != EOF) { switch (status) { @@ -450,10 +447,6 @@ main(int argc, char *argv[]) compile_only = 1; break; - case 'C': - flags |= CAPC_FLAG; - break; - case 'd': dir = optarg; break; @@ -466,12 +459,12 @@ main(int argc, char *argv[]) nofilter = 1; break; - case 'o': - outfile = optarg; + case 'N': + normalization = strdup(optarg); break; - case 'r': - flags |= R_FLAG; + case 'o': + outfile = optarg; break; case 's': @@ -482,10 +475,6 @@ main(int argc, char *argv[]) verbose = 1; break; - case 'w': - flags |= W_FLAG; - break; - case 'x': debug = 1; break; @@ -506,7 +495,16 @@ main(int argc, char *argv[]) usage(); report_time(NULL); - linebyline.init(flags); + + /* + * Check each normalizer to see if it fires. + */ + status = linebyline.init(normalization); + if (status != 0) + { + fprintf(stderr, "comparator: bad analyzer option\n"); + exit(1); + } /* special case if user gave exactly one tree */ if (!compile_only && argcount == 1) diff --git a/makefile b/makefile index 0a92d4e..485a089 100644 --- a/makefile +++ b/makefile @@ -7,12 +7,13 @@ include Makefile UNIX = /home/esr/src/unix SRC1 = sys3 SRC2 = 32v +OPTS="-N line-oriented, remove-braces, remove-whitespace" # Test that making several hash files at one go doesn't mess up. regress2: - comparator -C -d $(UNIX) $(SRC1) | dumpscf >test1-old.dump - comparator -C -d $(UNIX) $(SRC2) | dumpscf >test2-old.dump - comparator -C -c -d $(UNIX) $(SRC1) $(SRC2) + comparator $(OPTS) -d $(UNIX) $(SRC1) | dumpscf >test1-old.dump + comparator $(OPTS) -d $(UNIX) $(SRC2) | dumpscf >test2-old.dump + comparator $(OPTS) -c -d $(UNIX) $(SRC1) $(SRC2) dumpscf <$(SRC1).scf >test1.dump dumpscf <$(SRC2).scf >test2.dump diff -u test1-old.dump test1.dump @@ -20,11 +21,11 @@ regress2: # Test that making a report from SCF files works the same as from the trees regress3: - comparator -C -d $(UNIX) $(SRC1) $(SRC2) >test1.dump - comparator -C -c -d $(UNIX) $(SRC1) $(SRC2) + comparator $(OPTS) -d $(UNIX) $(SRC1) $(SRC2) >test1.dump + comparator $(OPTS) -c -d $(UNIX) $(SRC1) $(SRC2) comparator $(SRC1).scf $(SRC2).scf >test2.dump diff -u test1.dump test2.dump # Test SVr4 against Linux sco.ovl: - time comparator -v -C -d $(UNIX) src5r4 linux-2.4.19 >sco.ovl + time comparator -v $(OPTS) -d $(UNIX) src5r4 linux-2.4.19 >sco.ovl diff --git a/shred.h b/shred.h index df92ccc..d774e99 100644 --- a/shred.h +++ b/shred.h @@ -59,15 +59,11 @@ feature_t; struct analyzer_t /* structure describing a feature analyzer */ { - unsigned int flags; -#define CAPC_FLAG 0x01 -#define R_FLAG 0x02 -#define W_FLAG 0x04 - void (*init)(unsigned int); + int (*init)(unsigned char *); void (*mode)(int); feature_t *(*get)(const struct filehdr_t *, FILE *, linenum_t *); void (*free)(const char *); - void (*dump)(char *); + void (*dumpopt)(char *); }; /* mode bits for controlling the analyzer */ -- 2.43.0