From: Eric S. Raymond Date: Wed, 3 Sep 2003 03:43:41 +0000 (+0000) Subject: SHIF -> SCIF. X-Git-Tag: 1.0~47 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=d4847cf48c8710c7d2baf00048c3b1d3e9fd1e7f;p=comparator.git SHIF -> SCIF. --- diff --git a/Makefile b/Makefile index 36829ba..1a669bf 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,6 @@ clean: rm -f comparator shredtree.o md5.o shredcompare.o TEST1 TEST2 test-a: comparator - comparator -e -c test1 test2 + comparator test1 test2 test-b: comparator - comparator -e -c test3 test4 + comparator test3 test4 diff --git a/main.c b/main.c index c377d01..4dcce92 100644 --- a/main.c +++ b/main.c @@ -48,7 +48,7 @@ static void generate_shredfile(const char *tree, FILE *ofp) list = sorted_file_list(tree, &file_count); - fputs("#SHIF-A 1.0\n", ofp); + fputs("#SCIF-A 1.0\n", ofp); fputs("Generator-Program: comparator 1.0\n", ofp); fputs("Hash-Method: MD5\n", ofp); fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none"); @@ -125,29 +125,21 @@ main(int argc, char *argv[]) extern char *optarg; /* set by getopt */ extern int optind; /* set by getopt */ - int status, file_only, exec_direct; + int status, file_only, compile_only; - exec_direct = file_only = 0; - while ((status = getopt(argc, argv, "cd:efhs:w")) != EOF) + compile_only = file_only = 0; + while ((status = getopt(argc, argv, "cd:hs:w")) != EOF) { switch (status) { case 'c': - c_only = 1; + compile_only = 1; break; case 'd': chdir(optarg); break; - case 'e': - exec_direct = 1; - break; - - case 'f': - file_only = 1; - break; - case 's': shredsize = atoi(optarg); break; @@ -162,10 +154,9 @@ main(int argc, char *argv[]) case 'h': default: - fprintf(stderr,"usage: shredtree [-c] [-d dir ] [-s shredsize] [-w] [-x] path\n"); - fprintf(stderr," -c = check .c, .h, and .txt files only.\n"); + fprintf(stderr,"usage: shredtree [-c] [-d dir ] [-h] [-s shredsize] [-w] [-x] path\n"); + fprintf(stderr," -c = generate SCIF files\n"); fprintf(stderr," -d dir = change directory before digesting.\n"); - fprintf(stderr," -e = pass structures in core\n"); fprintf(stderr," -h = help (display this message).\n"); fprintf(stderr," -s size = set shred size (default %d)\n", shredsize); @@ -175,11 +166,11 @@ main(int argc, char *argv[]) } } - if (file_only) + if (compile_only) generate_shredfile(argv[optind], stdout); - else if (exec_direct) + else { - struct shif_t localblock; + struct scif_t localblock; localblock.hash_method = "MD5"; localblock.normalization = rws ? "remove_whitespace" : "none"; @@ -193,22 +184,25 @@ main(int argc, char *argv[]) report_time("Sort done"); emit_report(&localblock, sort_buffer, sort_count); } + +#if 0 else { int hashcount; struct sorthash_t *obarray; - struct shif_t *shiflist; + struct scif_t *sciflist; report_time(NULL); - shiflist = init_merge(argc-optind, argv+optind); - obarray = merge_hashes(shiflist, argc-optind, &hashcount); + sciflist = init_merge(argc-optind, argv+optind); + obarray = merge_hashes(sciflist, argc-optind, &hashcount); report_time("Hash merge done, %d entries", hashcount); sort_hashes(obarray, hashcount); report_time("Sort done"); - emit_report(shiflist, obarray, hashcount); + emit_report(sciflist, obarray, hashcount); } - exit(0); +#endif /* FOO */ + exit(0); } /* main.c ends here */ diff --git a/report.c b/report.c index 5a7735f..49edb8e 100644 --- a/report.c +++ b/report.c @@ -19,7 +19,7 @@ struct item dummy_item; static struct item *head = &dummy_item; -static struct shif_t *shiflist; +static struct scif_t *sciflist; struct range_t { @@ -50,23 +50,23 @@ static void file_intern(const char *file) head = new; } -struct shif_t *init_merge(int argc, char *argv[]) +struct scif_t *init_merge(int argc, char *argv[]) { - struct shif_t *sp; + struct scif_t *sp; /* set up metadata blocks for the hash files */ - shiflist = (struct shif_t *)calloc(sizeof(struct shif_t), argc); - for (sp = shiflist; sp < shiflist + argc; sp++) + sciflist = (struct scif_t *)calloc(sizeof(struct scif_t), argc); + for (sp = sciflist; sp < sciflist + argc; sp++) { char buf[BUFSIZ]; - sp->name = strdup(argv[sp - shiflist]); + sp->name = strdup(argv[sp - sciflist]); sp->fp = fopen(sp->name, "r"); fgets(buf, sizeof(buf), sp->fp); - if (strncmp(buf, "#SHIF-A ", 8)) + if (strncmp(buf, "#SCIF-A ", 8)) { fprintf(stderr, - "shredcompare: %s is not a SHIF-A file.", + "shredcompare: %s is not a SCIF-A file.", sp->name); exit(1); } @@ -94,9 +94,9 @@ struct shif_t *init_merge(int argc, char *argv[]) } /* consistency checks */ - for (sp = shiflist; sp < shiflist + argc-1; sp++) + for (sp = sciflist; sp < sciflist + argc-1; sp++) { - struct shif_t *next = sp + 1; + struct scif_t *next = sp + 1; if (strcmp(sp->normalization, next->normalization)) { @@ -122,13 +122,13 @@ struct shif_t *init_merge(int argc, char *argv[]) } } - return(shiflist); + return(sciflist); } -struct sorthash_t *merge_hashes(struct shif_t *shiflist, int shiflen, int *count) +struct sorthash_t *merge_hashes(struct scif_t *sciflist, int sciflen, int *count) /* merge hashes from specified files into an in-code list */ { - struct shif_t *sp; + struct scif_t *sp; long total; struct sorthash_t *obarray; int hashcount; @@ -136,7 +136,7 @@ struct sorthash_t *merge_hashes(struct shif_t *shiflist, int shiflen, int *count /* compute total data to be read, we'll use this for the progress meter */ total = 0; - for (sp = shiflist; sp < shiflist + shiflen; sp++) + for (sp = sciflist; sp < sciflist + sciflen; sp++) { stat(sp->name, &sb); total += sb.st_size - ftell(sp->fp); @@ -151,7 +151,7 @@ struct sorthash_t *merge_hashes(struct shif_t *shiflist, int shiflen, int *count /* read in all hashes */ hashcount = 0; - for (sp = shiflist; sp < shiflist + shiflen; sp++) + for (sp = sciflist; sp < sciflist + sciflen; sp++) { u_int32_t sectcount; @@ -414,7 +414,7 @@ static int sortmatch(void *a, void *b) return(0); } -void emit_report(struct shif_t *shif, +void emit_report(struct scif_t *scif, struct sorthash_t *obarray, int hashcount) { struct match_t *hitlist, *sorted, *match; @@ -446,11 +446,11 @@ void emit_report(struct shif_t *shif, hitlist = reduce_matches(obarray, hashcount); report_time("Reduction done"); - puts("#SHIF-B 1.0"); - printf("Hash-Method: %s\n", shif->hash_method); + puts("#SCIF-B 1.0"); + printf("Hash-Method: %s\n", scif->hash_method); puts("Merge-Program: comparator 1.0"); - printf("Normalization: %s\n", shif->normalization); - printf("Shred-Size: %d\n", shif->shred_size); + printf("Normalization: %s\n", scif->normalization); + printf("Shred-Size: %d\n", scif->shred_size); puts("%%"); /* we go through a little extra effort to emit a sorted list */ diff --git a/scf-standard.xml b/scf-standard.xml index 56e006b..8824e9a 100644 --- a/scf-standard.xml +++ b/scf-standard.xml @@ -1,4 +1,4 @@ -$Id$ +$Id: interchange,v 1.1 2003/09/03 03:04:51 esr Exp esr $ As of September 2003 there are several individuals and research groups engaged in building tools for doing analysis of similarities among source-code trees. @@ -118,9 +118,9 @@ Metadata tags short be written out in ASCII sort order of the tag. The ordering rules give every data-set a canonical form so that operations like diff(1) or comm(1) can give meaningful results. -SHIF-A: +SCIF-A: -This is the format for hash lists. The identifier is 'SHIF-A'. At version +This is the format for hash lists. The identifier is 'SCIF-A'. At version 1.0, the following metadata tags are defined: Normalization: (mandatory) @@ -171,9 +171,9 @@ This unpleasant binary format has been chosen to make processing as fast as possible, because shred data sets are quite large in holding down I/O is an important consideration. -SHIF-B: +SCIF-B: -This is the format for match lists. The identifier is 'SHIF-B'. At version +This is the format for match lists. The identifier is 'SCIF-B'. At version 1.0, the following metadata tags are defined: Filtering: (optional) @@ -205,6 +205,6 @@ then by end line. Ranges within range sets should be sorted by natural order. Range sets should be sorted by the natural order of their first member. -It is strongly recommended that when a SHIF-B file is generated from -SHIF-A files, the Normalization and Shred-Size headers should be copied +It is strongly recommended that when a SCIF-B file is generated from +SCIF-A files, the Normalization and Shred-Size headers should be copied into the result. diff --git a/shred.h b/shred.h index b35c095..1f5f946 100644 --- a/shred.h +++ b/shred.h @@ -30,7 +30,7 @@ struct sorthash_t char *file; }; -struct shif_t +struct scif_t { char *name; FILE *fp; @@ -50,11 +50,11 @@ extern int shredsize; /* shredtree.c functions */ extern char **sorted_file_list(const char *, int *); extern void shredfile(const char *, void (*hook)(struct hash_t, const char *)); -extern struct shif_t *init_merge(int argc, char *argv[]); -extern struct sorthash_t *merge_hashes(struct shif_t *shif, int shiflen, +extern struct scif_t *init_merge(int argc, char *argv[]); +extern struct sorthash_t *merge_hashes(struct scif_t *scif, int sciflen, int *count); extern void sort_hashes(struct sorthash_t *hashlist, int hashcount); -extern void emit_report(struct shif_t *, +extern void emit_report(struct scif_t *, struct sorthash_t *obarray, int hashcount); extern void report_time(char *legend, ...); diff --git a/shredtree.c b/shredtree.c index 39f5c1a..fd780dd 100644 --- a/shredtree.c +++ b/shredtree.c @@ -32,15 +32,36 @@ static int file_count; * *************************************************************************/ +#define MIN_PRINTABLE 0.9 + static int eligible(const char *file) /* is the specified file eligible to be compared? */ { + /* fast check for the most common suffixes */ #define endswith(suff) !strcmp(suff, file + strlen(file) - strlen(suff)) - if (c_only) - return endswith(".c") || endswith(".h") || endswith(".txt"); - else + if (endswith(".c") || endswith(".h") || endswith(".html")) return(1); + else if (endswith(".o")) + return(0); #undef endswith + else + { + FILE *fp = fopen(file, "r"); + char buf[BUFSIZ], *cp; + int printable; + + if (!fp || !fgets(buf, sizeof(buf)-1, fp)) + return(0); + fclose(fp); + + /* count printables */ + for (cp = buf; *cp; cp++) + if (isascii(*cp)) + printable++; + + /* are we over the critical percentage? */ + return (printable/strlen(buf) >= MIN_PRINTABLE); + } } static int normalize(char *buf)