From: Eric S. Raymond Date: Fri, 19 Sep 2003 11:11:56 +0000 (+0000) Subject: Analyzer refactoring is done. X-Git-Tag: 2.1~10 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=361fe06fdef706f4e584028312aa49a43ba8d06b;p=comparator.git Analyzer refactoring is done. --- diff --git a/Makefile b/Makefile index 74ee187..71e1ac0 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ DOCS = README NEWS comparator.xml scf-standard.xml COPYING EXTRAS = shredtree.py shredcompare.py TEST = test SOURCES = $(CODE) $(SCRIPTS) $(DOCS) $(EXTRAS) $(TEST) comparator.spec Makefile -CFLAGS = -O3 +CFLAGS = -g #-O3 LDFLAGS = all: comparator comparator.1 diff --git a/linebyline.c b/linebyline.c index 189c723..e915ef3 100644 --- a/linebyline.c +++ b/linebyline.c @@ -1,10 +1,21 @@ -/* filter.c -- filter lines for significance */ +/****************************************************************************** + +NAME: + linebyline.c -- line-oriented feature analyzer four source comparisons + +******************************************************************************/ #include #include #include #include +#include #include "shred.h" +/* control bits */ +int remove_braces = 0; +int remove_whitespace = 0; +int remove_comments = 0; + static char *c_patterns[] = { /* Idioms that don't convey any meaning in isolation */ "return [a-z]+", "return [01]+", @@ -38,7 +49,9 @@ static char *shell_patterns[] = { }; static regex_t shell_regexps[sizeof(shell_patterns)/sizeof(*shell_patterns)]; -void filter_init(void) +static linenum_t linecount; + +void analyzer_init(void) /* initialize line filtering */ { int i; @@ -63,7 +76,8 @@ static unsigned char active = 0; static regex_t *regexps; static int nregexps; -void filter_set(int mask) +void analyzer_mode(int mask) +/* set the analyzer mode -- meant to be called at the start of a file scan */ { active = mask; @@ -77,9 +91,66 @@ void filter_set(int mask) regexps = shell_regexps; nregexps = sizeof(shell_regexps)/sizeof(*shell_regexps); } + + /* this may have to be fixed someday! */ + linecount = 0; +} + +static int normalize(char *buf) +/* normalize a buffer in place, return 0 if it should be skipped */ +{ + if (remove_comments) + { + if (remove_braces) /* remove C comments */ + { + char *ss = strstr(buf, "//"); + + if (ss) + *ss = '\0'; + else + { + char *start = strstr(buf, "/*"), *end = strstr(buf, "*/"); + + if (start && end && start < end) + strcpy(start, end+2); + else if (start && !end) + *start = '\0'; + else if (end && !start) + *end = '\0'; + } + } + else + { + char *ss = strchr(buf, '#'); + + if (ss) + *ss = '\0'; + } + } + + if (remove_whitespace) /* strip whitespace, ignore blank lines */ + { + char *tp, *sp; + + for (tp = sp = buf; *sp; sp++) + if (*sp != ' ' && *sp != '\t' && *sp != '\n') + *tp++ = *sp; + *tp = '\0'; + } + if (remove_braces) /* strip C statement brackets */ + { + char *tp, *sp; + + for (tp = sp = buf; *sp; sp++) + if (*sp != '{' && *sp != '}') + *tp++ = *sp; + *tp = '\0'; + } + + return(buf[0]); } -int filter_pass(const char *line) +static int filter_pass(const char *line) /* return flags that apply to this line */ { if (active == 0) @@ -136,13 +207,77 @@ int filter_pass(const char *line) } } +feature_t *analyzer_get(const struct filehdr_t *file, FILE *fp, linenum_t *linenump) +/* get a feature (in this case, a line) from the input stream */ +{ + char buf[BUFSIZ]; + static feature_t feature; + + while(fgets(buf, sizeof(buf), fp) != NULL) + { + int braceline = 0; + + linecount++; + if (linecount >= MAX_LINENUM) + { + fprintf(stderr, "comparator: %s too large, only first %d lines will be compared.\n", file->name, MAX_LINENUM-1); + break; + } + + if (remove_braces) + { + char *cp; + + for (cp = buf; *cp && isspace(*cp); cp++) + continue; + braceline = (*cp == '}'); + } + if (!normalize(buf)) + { + /* + * What this is for is to include trailing C } lines in chunk + * listings even though we're ignoring them for comparison + * purposes (in order not to be fooled by variance in indent + * styles). This is a kluge, but it means we will capture + * entire C functions that differ only by brace placement. + */ + if (remove_braces && braceline) + extend_current_chunk(file->length); + continue; + } + + /* maybe we can get the file type from the first line? */ + if (linecount == 1 && buf[0] == '#') + if (strstr(buf, "sh")) + { + analyzer_mode(SHELL_CODE); + linecount = 1; + } + + /* time to return the feature */ + feature.text = strdup(buf); + feature.flags = filter_pass(buf) ? INSIGNIFICANT : 0; + *linenump = linecount; + return &feature; + } + + *linenump = linecount; + return(NULL); +} + +void analyzer_free(const char *text) +/* free a piece of storage previously handed to shredtree */ +{ + free(text); +} + #ifdef TEST int main(int argc, char *argv[]) { char buf[BUFSIZ]; - filter_init(); - filter_set(C_CODE); + analyzer_init(); + analyzer_mode(C_CODE); while(fgets(buf, BUFSIZ, stdin)) { printf("%02x: ", filter_pass(buf)); diff --git a/main.c b/main.c index 29b2727..8c8b364 100644 --- a/main.c +++ b/main.c @@ -522,7 +522,7 @@ main(int argc, char *argv[]) usage(); report_time(NULL); - filter_init(); + analyzer_init(); /* special case if user gave exactly one tree */ if (!compile_only && argcount == 1) diff --git a/shred.h b/shred.h index 1108730..7c27233 100644 --- a/shred.h +++ b/shred.h @@ -52,6 +52,13 @@ struct sorthash_t struct filehdr_t *file; }; +typedef struct +{ + char *text; + int flags; +} +feature_t; + /* control bits, meant to be set at startup */ extern int remove_braces; extern int remove_whitespace; @@ -74,10 +81,12 @@ extern int shredfile(struct filehdr_t *, void (*hook)(struct hash_t, struct filehdr_t *)); extern void sort_hashes(struct sorthash_t *hashlist, int hashcount); -/* filter.c functions */ -extern void filter_init(void); -extern void filter_set(int); -extern int filter_pass(const char *line); +/* linebyline.c functions */ +extern void analyzer_init(void); +extern void analyzer_mode(int); +extern feature_t *analyzer_get(const struct filehdr_t *, + FILE *, linenum_t *linenumber); +extern void analyzer_free(const char *); /* shredcompare.c functions */ extern int merge_compare(struct sorthash_t *obarray, int hashcount); diff --git a/shredtree.c b/shredtree.c index 4c91cfd..55d0740 100644 --- a/shredtree.c +++ b/shredtree.c @@ -19,9 +19,6 @@ #include "shred.h" /* control bits, meant to be set at startup */ -int remove_braces = 0; -int remove_whitespace = 0; -int remove_comments = 0; int shredsize = 3; struct item @@ -74,60 +71,6 @@ static int eligible(const char *file) } } -static int normalize(char *buf) -/* normalize a buffer in place, return 0 if it should be skipped */ -{ - if (remove_comments) - { - if (remove_braces) /* remove C comments */ - { - char *ss = strstr(buf, "//"); - - if (ss) - *ss = '\0'; - else - { - char *start = strstr(buf, "/*"), *end = strstr(buf, "*/"); - - if (start && end && start < end) - strcpy(start, end+2); - else if (start && !end) - *start = '\0'; - else if (end && !start) - *end = '\0'; - } - } - else - { - char *ss = strchr(buf, '#'); - - if (ss) - *ss = '\0'; - } - } - - if (remove_whitespace) /* strip whitespace, ignore blank lines */ - { - char *tp, *sp; - - for (tp = sp = buf; *sp; sp++) - if (*sp != ' ' && *sp != '\t' && *sp != '\n') - *tp++ = *sp; - *tp = '\0'; - } - if (remove_braces) /* strip C statement brackets */ - { - char *tp, *sp; - - for (tp = sp = buf; *sp; sp++) - if (*sp != '{' && *sp != '}') - *tp++ = *sp; - *tp = '\0'; - } - - return(buf[0]); -} - typedef struct { char *line; @@ -174,7 +117,9 @@ int shredfile(struct filehdr_t *file, FILE *fp; char buf[BUFSIZ]; int i, linecount, accepted; + linenum_t linenumber; shred *display; + feature_t *feature; if ((fp = fopen(file->name, "r")) == NULL) { @@ -185,77 +130,42 @@ int shredfile(struct filehdr_t *file, /* deduce what filtering type we should use */ #define endswith(suff) !strcmp(suff,file->name+strlen(file->name)-strlen(suff)) - filter_set(0); + analyzer_mode(0); if (endswith(".c") || endswith(".h")) - filter_set(C_CODE); + analyzer_mode(C_CODE); else if (endswith(".sh")) - filter_set(SHELL_CODE); + analyzer_mode(SHELL_CODE); #undef endswith display = (shred *)calloc(sizeof(shred), shredsize); - accepted = linecount = 0; - while(fgets(buf, sizeof(buf), fp) != NULL) + linenumber = accepted=0; + while ((feature = analyzer_get(file, fp, &linenumber))) { - int braceline = 0; - - linecount++; - if (linecount >= MAX_LINENUM) - { - fprintf(stderr, "comparator: %s too large, only first %d lines will be compared.\n", file->name, MAX_LINENUM-1); - break; - } - - if (remove_braces) - { - char *cp; - - for (cp = buf; *cp && isspace(*cp); cp++) - continue; - braceline = (*cp == '}'); - } - if (!normalize(buf)) - { - /* - * What this is for is to include trailing C } lines in chunk - * listings even though we're ignoring them for comparison - * purposes (in order not to be fooled by variance in indent - * styles). This is a kluge, but it means we will capture - * entire C functions that differ only by brace placement. - */ - if (remove_braces && braceline) - extend_current_chunk(file->length); - continue; - } accepted++; - /* maybe we can get the file type from the first line? */ - if (linecount == 1 && buf[0] == '#') - if (strstr(buf, "sh")) - filter_set(SHELL_CODE); - /* create new shred */ - display[shredsize-1].line = strdup(buf); - display[shredsize-1].start = linecount; - display[shredsize-1].flags = filter_pass(buf) ? INSIGNIFICANT : 0; + display[shredsize-1].line = feature->text; + display[shredsize-1].start = linenumber; + display[shredsize-1].flags = feature->flags; /* flush completed chunk */ if (accepted >= shredsize) - hook(emit_chunk(display, linecount), file); + hook(emit_chunk(display, linenumber), file); /* shreds in progress are shifted down */ - free(display[0].line); + analyzer_free(display[0].line); for (i=1; i < shredsize; i++) display[i-1] = display[i]; display[shredsize-1].line = NULL; display[shredsize-1].flags = 0; } if (accepted && accepted < shredsize) - hook(emit_chunk(display, linecount), file); + hook(emit_chunk(display, linenumber), file); free(display); fclose(fp); - return(linecount); + return(linenumber); } /*************************************************************************