SHIF -> SCIF.

diff --git a/Makefile b/Makefile
index 36829ba52e29a042d9148fdbf6054b780c4310d2..1a669bfa89b77ce42e6756cc49f2e03fa7f6792e 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -19,6 +19,6 @@ clean:
        rm -f comparator shredtree.o md5.o shredcompare.o TEST1 TEST2
 
 test-a: comparator
-       comparator -e -c test1 test2
+       comparator test1 test2
 test-b: comparator
-       comparator -e -c test3 test4
+       comparator test3 test4

diff --git a/main.c b/main.c
index c377d017cdf932f21f2435cd388d8a8364d8be5a..4dcce92f3106573852af9e91873de965db118a18 100644 (file)
--- a/main.c
+++ b/main.c
@@ -48,7 +48,7 @@ static void generate_shredfile(const char *tree, FILE *ofp)
 
     list = sorted_file_list(tree, &file_count);
 
-    fputs("#SHIF-A 1.0\n", ofp);
+    fputs("#SCIF-A 1.0\n", ofp);
     fputs("Generator-Program: comparator 1.0\n", ofp);
     fputs("Hash-Method: MD5\n", ofp);
     fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none");
@@ -125,29 +125,21 @@ main(int argc, char *argv[])
     extern char        *optarg;        /* set by getopt */
     extern int optind;         /* set by getopt */
 
-    int status, file_only, exec_direct;
+    int status, file_only, compile_only;
 
-    exec_direct = file_only = 0;
-    while ((status = getopt(argc, argv, "cd:efhs:w")) != EOF)
+    compile_only = file_only = 0;
+    while ((status = getopt(argc, argv, "cd:hs:w")) != EOF)
     {
        switch (status)
        {
        case 'c':
-           c_only = 1;
+           compile_only = 1;
            break;
 
        case 'd':
            chdir(optarg);
            break;
 
-       case 'e':
-           exec_direct = 1;
-           break;
-
-       case 'f':
-           file_only = 1;
-           break;
-
        case 's':
            shredsize = atoi(optarg);
            break;
@@ -162,10 +154,9 @@ main(int argc, char *argv[])
 
        case 'h':
        default:
-           fprintf(stderr,"usage: shredtree [-c] [-d dir ] [-s shredsize] [-w] [-x] path\n");
-           fprintf(stderr,"  -c      = check .c, .h, and .txt files only.\n");
+           fprintf(stderr,"usage: shredtree [-c] [-d dir ] [-h] [-s shredsize] [-w] [-x] path\n");
+           fprintf(stderr,"  -c      = generate SCIF files\n");
            fprintf(stderr,"  -d dir  = change directory before digesting.\n");
-           fprintf(stderr,"  -e      = pass structures in core\n");
            fprintf(stderr,"  -h      = help (display this message).\n");
            fprintf(stderr,"  -s size = set shred size (default %d)\n",
                    shredsize);
@@ -175,11 +166,11 @@ main(int argc, char *argv[])
        }
     }
 
-    if (file_only)
+    if (compile_only)
        generate_shredfile(argv[optind], stdout);
-    else if (exec_direct)
+    else
     {
-       struct shif_t   localblock;
+       struct scif_t   localblock;
 
        localblock.hash_method = "MD5";
        localblock.normalization = rws ? "remove_whitespace" : "none";
@@ -193,22 +184,25 @@ main(int argc, char *argv[])
        report_time("Sort done");
        emit_report(&localblock, sort_buffer, sort_count);      
     }
+
+#if 0
     else
     {
        int     hashcount;
        struct sorthash_t *obarray;
-       struct shif_t   *shiflist;
+       struct scif_t   *sciflist;
 
        report_time(NULL);
-       shiflist = init_merge(argc-optind, argv+optind);
-       obarray = merge_hashes(shiflist, argc-optind, &hashcount);
+       sciflist = init_merge(argc-optind, argv+optind);
+       obarray = merge_hashes(sciflist, argc-optind, &hashcount);
        report_time("Hash merge done, %d entries", hashcount);
        sort_hashes(obarray, hashcount);
        report_time("Sort done");
-       emit_report(shiflist, obarray, hashcount);
+       emit_report(sciflist, obarray, hashcount);
     }
-    exit(0);
+#endif /* FOO */
 
+    exit(0);
 }
 
 /* main.c ends here */

diff --git a/report.c b/report.c
index 5a7735f2aa8509f7f0f8102d48c1896c09972681..49edb8e887897555680438306b73d15d78f6b69f 100644 (file)
--- a/report.c
+++ b/report.c
@@ -19,7 +19,7 @@ struct item
 dummy_item;
 static struct item *head = &dummy_item;
 
-static struct shif_t *shiflist;
+static struct scif_t *sciflist;
 
 struct range_t
 {
@@ -50,23 +50,23 @@ static void file_intern(const char *file)
     head = new;
 }
 
-struct shif_t *init_merge(int argc, char *argv[])
+struct scif_t *init_merge(int argc, char *argv[])
 {
-    struct shif_t *sp;
+    struct scif_t *sp;
 
     /* set up metadata blocks for the hash files */
-    shiflist = (struct shif_t *)calloc(sizeof(struct shif_t), argc);
-    for (sp = shiflist; sp < shiflist + argc; sp++)
+    sciflist = (struct scif_t *)calloc(sizeof(struct scif_t), argc);
+    for (sp = sciflist; sp < sciflist + argc; sp++)
     {
        char    buf[BUFSIZ];
 
-       sp->name = strdup(argv[sp - shiflist]);
+       sp->name = strdup(argv[sp - sciflist]);
        sp->fp   = fopen(sp->name, "r");
        fgets(buf, sizeof(buf), sp->fp);
-       if (strncmp(buf, "#SHIF-A ", 8))
+       if (strncmp(buf, "#SCIF-A ", 8))
        {
            fprintf(stderr, 
-                   "shredcompare: %s is not a SHIF-A file.", 
+                   "shredcompare: %s is not a SCIF-A file.", 
                    sp->name);
            exit(1);
        }
@@ -94,9 +94,9 @@ struct shif_t *init_merge(int argc, char *argv[])
     }
 
     /* consistency checks */
-    for (sp = shiflist; sp < shiflist + argc-1; sp++)
+    for (sp = sciflist; sp < sciflist + argc-1; sp++)
     {
-       struct shif_t   *next = sp + 1;
+       struct scif_t   *next = sp + 1;
 
        if (strcmp(sp->normalization, next->normalization))
        {
@@ -122,13 +122,13 @@ struct shif_t *init_merge(int argc, char *argv[])
        }
     }
 
-    return(shiflist);
+    return(sciflist);
 }
 
-struct sorthash_t *merge_hashes(struct shif_t *shiflist, int shiflen, int *count)
+struct sorthash_t *merge_hashes(struct scif_t *sciflist, int sciflen, int *count)
 /* merge hashes from specified files into an in-code list */
 {
-    struct shif_t *sp;
+    struct scif_t *sp;
     long total;
     struct sorthash_t *obarray;
     int hashcount;
@@ -136,7 +136,7 @@ struct sorthash_t *merge_hashes(struct shif_t *shiflist, int shiflen, int *count
 
     /* compute total data to be read, we'll use this for the progress meter */
     total = 0;
-    for (sp = shiflist; sp < shiflist + shiflen; sp++)
+    for (sp = sciflist; sp < sciflist + sciflen; sp++)
     {
        stat(sp->name, &sb);
        total += sb.st_size - ftell(sp->fp);
@@ -151,7 +151,7 @@ struct sorthash_t *merge_hashes(struct shif_t *shiflist, int shiflen, int *count
 
     /* read in all hashes */
     hashcount = 0;
-    for (sp = shiflist; sp < shiflist + shiflen; sp++)
+    for (sp = sciflist; sp < sciflist + sciflen; sp++)
     {
        u_int32_t       sectcount;
 
@@ -414,7 +414,7 @@ static int sortmatch(void *a, void *b)
     return(0);
 }
 
-void emit_report(struct shif_t *shif, 
+void emit_report(struct scif_t *scif, 
                 struct sorthash_t *obarray, int hashcount)
 {
     struct match_t *hitlist, *sorted, *match;
@@ -446,11 +446,11 @@ void emit_report(struct shif_t *shif,
     hitlist = reduce_matches(obarray, hashcount);
     report_time("Reduction done");
 
-    puts("#SHIF-B 1.0");
-    printf("Hash-Method: %s\n", shif->hash_method);
+    puts("#SCIF-B 1.0");
+    printf("Hash-Method: %s\n", scif->hash_method);
     puts("Merge-Program: comparator 1.0");
-    printf("Normalization: %s\n", shif->normalization);
-    printf("Shred-Size: %d\n", shif->shred_size);
+    printf("Normalization: %s\n", scif->normalization);
+    printf("Shred-Size: %d\n", scif->shred_size);
     puts("%%");
 
     /* we go through a little extra effort to emit a sorted list */

diff --git a/scf-standard.xml b/scf-standard.xml
index 56e006b30c160b504c129d795c408e5dbc5d9570..8824e9a9750e604e58c59b67976c8f44f144c829 100644 (file)
--- a/scf-standard.xml
+++ b/scf-standard.xml
@@ -1,4 +1,4 @@
-$Id$
+$Id: interchange,v 1.1 2003/09/03 03:04:51 esr Exp esr $
 
 As of September 2003 there are several individuals and research groups engaged
 in building tools for doing analysis of similarities among source-code trees.
@@ -118,9 +118,9 @@ Metadata tags short be written out in ASCII sort order of the tag.
 The ordering rules give every data-set a canonical form so that
 operations like diff(1) or comm(1) can give meaningful results.
 
-SHIF-A:
+SCIF-A:
 
-This is the format for hash lists.  The identifier is 'SHIF-A'.  At version
+This is the format for hash lists.  The identifier is 'SCIF-A'.  At version
 1.0, the following metadata tags are defined:
 
 Normalization: (mandatory)
@@ -171,9 +171,9 @@ This unpleasant binary format has been chosen to make processing as
 fast as possible, because shred data sets are quite large in holding
 down I/O is an important consideration.
 
-SHIF-B:
+SCIF-B:
 
-This is the format for match lists. The identifier is 'SHIF-B'.  At version
+This is the format for match lists. The identifier is 'SCIF-B'.  At version
 1.0, the following metadata tags are defined:
 
 Filtering: (optional)
@@ -205,6 +205,6 @@ then by end line.  Ranges within range sets should be sorted by natural
 order.  Range sets should be sorted by the natural order of their first
 member.
 
-It is strongly recommended that when a SHIF-B file is generated from
-SHIF-A files, the Normalization and Shred-Size headers should be copied
+It is strongly recommended that when a SCIF-B file is generated from
+SCIF-A files, the Normalization and Shred-Size headers should be copied
 into the result.

diff --git a/shred.h b/shred.h
index b35c0959f777179fdfdf0a7593e6de64b07f56a4..1f5f946e0eb3e97e36ba90db4b73b2f2e7edf8c7 100644 (file)
--- a/shred.h
+++ b/shred.h
@@ -30,7 +30,7 @@ struct sorthash_t
     char               *file;
 };
 
-struct shif_t
+struct scif_t
 {
     char       *name;
     FILE       *fp;
@@ -50,11 +50,11 @@ extern int shredsize;
 /* shredtree.c functions */
 extern char **sorted_file_list(const char *, int *);
 extern void shredfile(const char *, void (*hook)(struct hash_t, const char *));
-extern struct shif_t *init_merge(int argc, char *argv[]);
-extern struct sorthash_t *merge_hashes(struct shif_t *shif, int shiflen, 
+extern struct scif_t *init_merge(int argc, char *argv[]);
+extern struct sorthash_t *merge_hashes(struct scif_t *scif, int sciflen, 
                                       int *count);
 extern void sort_hashes(struct sorthash_t *hashlist, int hashcount);
-extern void emit_report(struct shif_t *, 
+extern void emit_report(struct scif_t *, 
                        struct sorthash_t *obarray, int hashcount);
 extern void report_time(char *legend, ...);
 

diff --git a/shredtree.c b/shredtree.c
index 39f5c1ae12dda7be6692ce8cec6f971289ee928b..fd780dd93f68f9d2713772d1945bc13c542143cc 100644 (file)
--- a/shredtree.c
+++ b/shredtree.c
@@ -32,15 +32,36 @@ static int file_count;
  *
  *************************************************************************/
 
+#define MIN_PRINTABLE  0.9
+
 static int eligible(const char *file)
 /* is the specified file eligible to be compared? */ 
 {
+    /* fast check for the most common suffixes */
 #define endswith(suff) !strcmp(suff, file + strlen(file) - strlen(suff))
-    if (c_only)
-       return endswith(".c") || endswith(".h") || endswith(".txt");
-    else
+    if (endswith(".c") || endswith(".h") || endswith(".html"))
        return(1);
+    else if (endswith(".o"))
+       return(0);
 #undef endswith
+    else
+    {
+       FILE    *fp = fopen(file, "r");
+       char    buf[BUFSIZ], *cp;
+       int     printable;
+
+       if (!fp || !fgets(buf, sizeof(buf)-1, fp))
+           return(0);
+       fclose(fp);
+
+       /* count printables */
+       for (cp = buf; *cp; cp++)
+           if (isascii(*cp))
+               printable++;
+
+       /* are we over the critical percentage? */
+       return (printable/strlen(buf) >= MIN_PRINTABLE);
+    }
 }
 
 static int normalize(char *buf)