From 7987cbaffbeca21491c78791500915460890e41d Mon Sep 17 00:00:00 2001
From: "Eric S. Raymond" <esr@thyrsus.com>
Date: Wed, 3 Sep 2003 07:51:11 +0000
Subject: [PATCH] Rebuilt to put all the SCF knowledge in main.c.

---
 main.c           | 221 +++++++++++++++++++++++++++++++++++++----------
 report.c         | 192 ++++++++--------------------------------
 scf-standard.xml |  16 ++--
 shred.h          |  24 ++---
 4 files changed, 229 insertions(+), 224 deletions(-)

diff --git a/main.c b/main.c
index 4dcce92..b9f7513 100644
--- a/main.c
+++ b/main.c
@@ -3,12 +3,40 @@
 #include <stdio.h>
 #include <unistd.h>
 #include <stdarg.h>
+#include <string.h>
 #include "shred.h"
 
+struct scf_t
+{
+    char	*name;
+    FILE	*fp;
+    char	*normalization;
+    int		shred_size;
+    char	*hash_method;
+    char	*generator_program;
+    struct scf_t *next;
+};
+static struct scf_t dummy_scf, *scf_head = &dummy_scf;
+
 static int chunk_count, sort_count;
 static struct hash_t *chunk_buffer;
 static struct sorthash_t *sort_buffer;
 
+static int is_scf_file(const char *file)
+/* is the specified file an SCF hash list? */
+{
+    char	buf[10];
+    FILE	*fp = fopen(file, "r");
+
+    if (!fgets(buf, sizeof(buf), fp))
+    {
+	fclose(fp);
+	return(0);
+    }
+    fclose(fp);
+    return(!strncmp(buf, "#SCF-A ", 7));
+}
+
 /* 
  * We differentiate these two functions in order to avaid the memory
  * overhead of the extra (char *) when assembling shred lists to be written
@@ -28,7 +56,7 @@ static void filehook(struct hash_t hash, const char *file)
 }
 
 
-static void corehook(struct hash_t hash, const char *file)
+void corehook(struct hash_t hash, const char *file)
 /* hook to store hash and file */
 {
     sort_buffer = (struct sorthash_t *)realloc(sort_buffer, 
@@ -48,7 +76,7 @@ static void generate_shredfile(const char *tree, FILE *ofp)
 
     list = sorted_file_list(tree, &file_count);
 
-    fputs("#SCIF-A 1.0\n", ofp);
+    fputs("#SCF-A 1.0\n", ofp);
     fputs("Generator-Program: comparator 1.0\n", ofp);
     fputs("Hash-Method: MD5\n", ofp);
     fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none");
@@ -76,30 +104,69 @@ static void generate_shredfile(const char *tree, FILE *ofp)
     }
 }
 
-static void generate_shredlist(int argc, char *argv[])
-/* generate an in-core list of sorthash structures */
+static void merge_tree(char *tree)
+/* add to the in-core list of sorthash structures from a tree */
+{
+    char	**place, **list;
+    int	oldcount, file_count;
+
+    oldcount = sort_count;
+    fprintf(stderr, "%% Reading %s...", tree);
+    list = sorted_file_list(tree, &file_count);
+    for (place = list; place < list + file_count; place++)
+	shredfile(*place, corehook);
+    fprintf(stderr, "%d entries\n", sort_count - oldcount);
+    free(list);
+}
+
+static void init_scf(char *file)
+/* add to the in-core list of sorthash structures from a SCF file */
 {
-    int	i, file_count;
+    struct scf_t	*new;
+    char	buf[BUFSIZ];
 
-    sort_buffer = (struct sorthash_t*)calloc(sizeof(struct sorthash_t),1);
-    sort_count = 0;
+    new = (struct scf_t *)malloc(sizeof(struct scf_t));
 
-    for (i = 0; i < argc; i++)
+    /* read in the SCF metadata block and add it to the in-core list */
+    new->name = strdup(file);
+    new->fp   = fopen(new->name, "r");
+    fgets(buf, sizeof(buf), new->fp);
+    if (strncmp(buf, "#SCF-A ", 8))
     {
-	char	**place, **list;
-	int	oldcount;
-
-	oldcount = sort_count;
-	fprintf(stderr, "%% Reading %s...", argv[i]);
-	list = sorted_file_list(argv[i], &file_count);
-	for (place = list; place < list + file_count; place++)
-	    shredfile(*place, corehook);
-	fprintf(stderr, "%d entries\n", sort_count - oldcount);
-	free(list);
+	fprintf(stderr, 
+		"shredcompare: %s is not a SCF-A file.", 
+		new->name);
+	exit(1);
     }
+    while (fgets(buf, sizeof(buf), new->fp) != NULL)
+    {
+	char	*value;
+
+	if (!strcmp(buf, "%%\n"))
+	    break;
+	value = strchr(buf, ':');
+	*value++ = '\0';
+	while(*value == ' ')
+	    value++;
+	strchr(value, '\n')[0] = '\0';
+
+	if (!strcmp(buf, "Normalization"))
+	    new->normalization = strdup(value);
+	else if (!strcmp(buf, "Shred-Size"))
+	    new->shred_size = atoi(value);
+	else if (!strcmp(buf, "Hash-Method"))
+	    new->hash_method = strdup(value);
+	else if (!strcmp(buf, "Generator-Program"))
+	    new->generator_program = strdup(value);
+    }
+
+    new->name = strdup(file);
+    new->next = scf_head;
+    scf_head = new;
 }
 
 void report_time(char *legend, ...)
+/* report on time since last report_mark */
 {
     static time_t mark_time;
     time_t endtime = time(NULL);
@@ -126,6 +193,7 @@ main(int argc, char *argv[])
     extern int	optind;		/* set by getopt */
 
     int status, file_only, compile_only;
+    struct scf_t	*scf;
 
     compile_only = file_only = 0;
     while ((status = getopt(argc, argv, "cd:hs:w")) != EOF)
@@ -155,7 +223,7 @@ main(int argc, char *argv[])
 	case 'h':
 	default:
 	    fprintf(stderr,"usage: shredtree [-c] [-d dir ] [-h] [-s shredsize] [-w] [-x] path\n");
-	    fprintf(stderr,"  -c      = generate SCIF files\n");
+	    fprintf(stderr,"  -c      = generate SCF files\n");
 	    fprintf(stderr,"  -d dir  = change directory before digesting.\n");
 	    fprintf(stderr,"  -h      = help (display this message).\n");
 	    fprintf(stderr,"  -s size = set shred size (default %d)\n",
@@ -166,41 +234,104 @@ main(int argc, char *argv[])
 	}
     }
 
-    if (compile_only)
-	generate_shredfile(argv[optind], stdout);
-    else
+    if (!compile_only)
     {
-	struct scif_t	localblock;
+	sort_buffer = (struct sorthash_t*)calloc(sizeof(struct sorthash_t),1);
+	sort_count = 0;
+    }
 
-	localblock.hash_method = "MD5";
-	localblock.normalization = rws ? "remove_whitespace" : "none";
-	localblock.shred_size = shredsize;
-	localblock.generator_program = "comparator 1.0";
+    report_time(NULL);
+    for(; optind < argc; optind++)
+    {
+	char	*source = argv[optind];
 
-	report_time(NULL);
-	generate_shredlist(argc-optind, argv+optind);
-	report_time("Hash merge done, %d entries", sort_count);
-	sort_hashes(sort_buffer, sort_count);
-	report_time("Sort done");
-	emit_report(&localblock, sort_buffer, sort_count);	
+	if (is_scf_file(source))
+	    init_scf(source);
+	else if (compile_only)
+	{
+	    char	*outfile = alloca(strlen(source) + 4);
+	    FILE	*ofp;
+
+	    strcpy(outfile, source);
+	    strcat(outfile, ".scf");
+	    ofp = fopen(outfile, "w");
+	    if (!ofp)
+	    {
+		fprintf(stderr, "comparator: couldn't open output file %s\n",
+			outfile);
+		exit(1);
+	    }
+	    generate_shredfile(source, ofp);
+	    fclose(ofp);
+	}
+	else
+	    merge_tree(source);
     }
 
-#if 0
+    /* if there were no SCFs, create a dummy one */
+    if (scf_head == &dummy_scf)
+    {
+	dummy_scf.hash_method = "MD5";
+	dummy_scf.normalization = rws ? "remove_whitespace" : "none";
+	dummy_scf.shred_size = shredsize;
+	dummy_scf.generator_program = "comparator 1.0";
+    }
     else
     {
-	int	hashcount;
-	struct sorthash_t *obarray;
-	struct scif_t	*sciflist;
-
-	report_time(NULL);
-	sciflist = init_merge(argc-optind, argv+optind);
-	obarray = merge_hashes(sciflist, argc-optind, &hashcount);
-	report_time("Hash merge done, %d entries", hashcount);
-	sort_hashes(obarray, hashcount);
+	/* consistency checks on the SCFs */
+	for (scf = scf_head; scf->next; scf = scf->next)
+	{
+	    struct scf_t	*next = scf + 1;
+
+	    if (strcmp(scf->normalization, next->normalization))
+	    {
+		fprintf(stderr, 
+			"shredcompare: normalizations of %s and %s don't match\n",
+			scf->name, next->name);
+		exit(1);
+	    }
+	    else if (scf->shred_size != next->shred_size)
+	    {
+		fprintf(stderr, 
+			"shredcompare: shred sizes of %s and %s don't match\n",
+			scf->name, next->name);
+		exit(1);
+
+	    }
+	    else if (strcmp(scf->hash_method, next->hash_method))
+	    {
+		fprintf(stderr, 
+			"shredcompare: hash methods of %s and %s don't match\n",
+			scf->name, next->name);
+		exit(1);
+	    }
+	}
+
+	/* finish reading in all SCFs */
+	for (scf = scf_head; scf->next; scf = scf->next)
+	{
+	    merge_scf(scf->name, scf->fp);
+	    fclose(scf->fp);
+	}
+
+    }
+
+    if (!compile_only)
+    {
+	/* now we're ready to emit the report */
+	puts("#SCF-B 1.0");
+	printf("Hash-Method: %s\n", scf_head->hash_method);
+	puts("Merge-Program: comparator 1.0");
+	printf("Normalization: %s\n", scf_head->normalization);
+	printf("Shred-Size: %d\n", scf_head->shred_size);
+	puts("%%");
+
+	report_time("Hash merge done, %d entries", sort_count);
+	sort_hashes(sort_buffer, sort_count);
 	report_time("Sort done");
-	emit_report(sciflist, obarray, hashcount);
+
+	emit_report(sort_buffer, sort_count);
     }
-#endif /* FOO */
 
     exit(0);
 }
diff --git a/report.c b/report.c
index 49edb8e..d056a7b 100644
--- a/report.c
+++ b/report.c
@@ -19,7 +19,7 @@ struct item
 dummy_item;
 static struct item *head = &dummy_item;
 
-static struct scif_t *sciflist;
+static struct scf_t *scflist;
 
 struct range_t
 {
@@ -39,156 +39,46 @@ struct match_t
 dummy_match;
 static struct match_t *reduced = &dummy_match;
 
-static void file_intern(const char *file)
-/* stash the name of a file */
-{
-    struct item *new;
-
-    new = (struct item *)malloc(sizeof(struct item));
-    new->file = strdup(file);
-    new->next = head;
-    head = new;
-}
-
-struct scif_t *init_merge(int argc, char *argv[])
-{
-    struct scif_t *sp;
-
-    /* set up metadata blocks for the hash files */
-    sciflist = (struct scif_t *)calloc(sizeof(struct scif_t), argc);
-    for (sp = sciflist; sp < sciflist + argc; sp++)
-    {
-	char	buf[BUFSIZ];
-
-	sp->name = strdup(argv[sp - sciflist]);
-	sp->fp   = fopen(sp->name, "r");
-	fgets(buf, sizeof(buf), sp->fp);
-	if (strncmp(buf, "#SCIF-A ", 8))
-	{
-	    fprintf(stderr, 
-		    "shredcompare: %s is not a SCIF-A file.", 
-		    sp->name);
-	    exit(1);
-	}
-	while (fgets(buf, sizeof(buf), sp->fp) != NULL)
-	{
-	    char	*value;
-
-	    if (!strcmp(buf, "%%\n"))
-		break;
-	    value = strchr(buf, ':');
-	    *value++ = '\0';
-	    while(*value == ' ')
-		value++;
-	    strchr(value, '\n')[0] = '\0';
-
-	    if (!strcmp(buf, "Normalization"))
-		sp->normalization = strdup(value);
-	    else if (!strcmp(buf, "Shred-Size"))
-		sp->shred_size = atoi(value);
-	    else if (!strcmp(buf, "Hash-Method"))
-		sp->hash_method = strdup(value);
-	    else if (!strcmp(buf, "Generator-Program"))
-		sp->generator_program = strdup(value);
-	}
-    }
-
-    /* consistency checks */
-    for (sp = sciflist; sp < sciflist + argc-1; sp++)
-    {
-	struct scif_t	*next = sp + 1;
-
-	if (strcmp(sp->normalization, next->normalization))
-	{
-	    fprintf(stderr, 
-		    "shredcompare: normalizations of %s and %s don't match\n",
-		    sp->name, next->name);
-	    exit(1);
-	}
-	else if (sp->shred_size != next->shred_size)
-	{
-	    fprintf(stderr, 
-		    "shredcompare: shred sizes of %s and %s don't match\n",
-		    sp->name, next->name);
-	    exit(1);
-
-	}
-	else if (strcmp(sp->hash_method, next->hash_method))
-	{
-	    fprintf(stderr, 
-		    "shredcompare: hash methods of %s and %s don't match\n",
-		    sp->name, next->name);
-	    exit(1);
-	}
-    }
-
-    return(sciflist);
-}
-
-struct sorthash_t *merge_hashes(struct scif_t *sciflist, int sciflen, int *count)
+void merge_scf(const char *name, FILE *fp)
 /* merge hashes from specified files into an in-code list */
 {
-    struct scif_t *sp;
-    long total;
-    struct sorthash_t *obarray;
-    int hashcount;
-    struct stat sb;
-
-    /* compute total data to be read, we'll use this for the progress meter */
-    total = 0;
-    for (sp = sciflist; sp < sciflist + sciflen; sp++)
-    {
-	stat(sp->name, &sb);
-	total += sb.st_size - ftell(sp->fp);
-    }
-    obarray=(struct sorthash_t *)calloc(sizeof(struct sorthash_t),
-					   total/sizeof(struct hash_t));
-    if (!obarray)
-    {
-	fprintf(stderr, "shredcompare: insufficient memory\n");
-	exit(1);
-    }
+    u_int32_t	sectcount;
+    int hashcount = 0;
 
-    /* read in all hashes */
-    hashcount = 0;
-    for (sp = sciflist; sp < sciflist + sciflen; sp++)
+    fprintf(stderr, "%% Reading %s...   ", name);
+    fread(&sectcount, sizeof(u_int32_t), 1, fp);
+    sectcount = ntohl(sectcount);
+    while (sectcount--)
     {
-	u_int32_t	sectcount;
-
-	fprintf(stderr, "Reading %s...   ", sp->name);
-	fread(&sectcount, sizeof(u_int32_t), 1, sp->fp);
-	sectcount = ntohl(sectcount);
-	while (sectcount--)
+	char	buf[BUFSIZ];
+	linenum_t	chunks;
+	struct item *new;
+
+	fgets(buf, sizeof(buf), fp);
+	*strchr(buf, '\n') = '\0';
+	new = (struct item *)malloc(sizeof(struct item));
+	new->file = strdup(name);
+	new->next = head;
+	head = new;
+	fread(&chunks, sizeof(linenum_t), 1, fp);
+	chunks = FROMNET(chunks);
+
+	while (chunks--)
 	{
-	    char	buf[BUFSIZ];
-	    linenum_t	chunks;
-
-	    fgets(buf, sizeof(buf), sp->fp);
-	    *strchr(buf, '\n') = '\0';
-	    file_intern(buf);			/* real work done here */
-	    fread(&chunks, sizeof(linenum_t), 1, sp->fp);
-	    chunks = FROMNET(chunks);
-
-	    while (chunks--)
-	    {
-		struct hash_t	this;
-
-		fread(&this, sizeof(struct hash_t), 1, sp->fp);
-		this.start = FROMNET(this.start);
-		this.end = FROMNET(this.end);
-		obarray[hashcount].file = head->file;
-		obarray[hashcount].hash = this;
-		hashcount++;
-		sp->hashcount++;
-		if (hashcount % 10000 == 0)
-		    fprintf(stderr, "\b\b\b%02.0f%%", (ftell(sp->fp) / (total * 0.01)));
-	    }
+	    struct hash_t	this;
+
+	    fread(&this.hash, sizeof(struct hash_t), 1, fp);
+	    this.start = FROMNET(this.start);
+	    this.end = FROMNET(this.end);
+	    corehook(this, head->file);
+	    hashcount++;
+#ifdef FIXME
+	    if (hashcount % 10000 == 0)
+		fprintf(stderr,"\b\b\b%02.0f%%",(ftell(fp) / (total * 0.01)));
+#endif
 	}
-	fprintf(stderr, "\b\b\b100%%...done, %d entries\n", sp->hashcount);
     }
-
-    *count = hashcount;
-    return obarray;
+    fprintf(stderr, "\b\b\b100%%...done, %d entries\n", hashcount);
 }
 
 static int merge_ranges(struct range_t *p, struct range_t *q, int nmatches)
@@ -414,13 +304,14 @@ static int sortmatch(void *a, void *b)
     return(0);
 }
 
-void emit_report(struct scif_t *scif, 
-		 struct sorthash_t *obarray, int hashcount)
+void emit_report(struct sorthash_t *obarray, int hashcount)
 {
     struct match_t *hitlist, *sorted, *match;
     int i, matchcount;
 
 #ifdef ODEBUG
+    struct sorthash_t	*np;
+
     for (np = obarray; np < obarray + hashcount; np++)
 	printf("%d: %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x %s:%d:%d\n", 
 	       np-obarray, 
@@ -446,13 +337,6 @@ void emit_report(struct scif_t *scif,
     hitlist = reduce_matches(obarray, hashcount);
     report_time("Reduction done");
 
-    puts("#SCIF-B 1.0");
-    printf("Hash-Method: %s\n", scif->hash_method);
-    puts("Merge-Program: comparator 1.0");
-    printf("Normalization: %s\n", scif->normalization);
-    printf("Shred-Size: %d\n", scif->shred_size);
-    puts("%%");
-
     /* we go through a little extra effort to emit a sorted list */
     matchcount = 0;
     for (match = hitlist; match->next; match = match->next)
@@ -476,7 +360,7 @@ void emit_report(struct scif_t *scif,
 
 	    printf("%s:%d:%d\n",  rp->file, rp->start, rp->end);
 	}
-	printf("-\n");
+	printf("%%%%\n");
     }
 }
 
diff --git a/scf-standard.xml b/scf-standard.xml
index 8824e9a..a30ce0a 100644
--- a/scf-standard.xml
+++ b/scf-standard.xml
@@ -1,4 +1,4 @@
-$Id: interchange,v 1.1 2003/09/03 03:04:51 esr Exp esr $
+$Id: interchange,v 1.2 2003/09/03 03:43:41 esr Exp esr $
 
 As of September 2003 there are several individuals and research groups engaged
 in building tools for doing analysis of similarities among source-code trees.
@@ -100,7 +100,7 @@ We also need a common format for match lists.  The format should be
 able to describe what reduction and filtering steps were performed.
 
 The remainder of this document describes file formats to fill these needs.
-These formats will be known as SCIF (source Comparison Interchange Format)
+These formats will be known as SCF (source Comparison Interchange Format)
 files, pronounced "skiff".
 
 Each format consists of three sections: an identification line,
@@ -118,9 +118,9 @@ Metadata tags short be written out in ASCII sort order of the tag.
 The ordering rules give every data-set a canonical form so that
 operations like diff(1) or comm(1) can give meaningful results.
 
-SCIF-A:
+SCF-A:
 
-This is the format for hash lists.  The identifier is 'SCIF-A'.  At version
+This is the format for hash lists.  The identifier is 'SCF-A'.  At version
 1.0, the following metadata tags are defined:
 
 Normalization: (mandatory)
@@ -171,9 +171,9 @@ This unpleasant binary format has been chosen to make processing as
 fast as possible, because shred data sets are quite large in holding
 down I/O is an important consideration.
 
-SCIF-B:
+SCF-B:
 
-This is the format for match lists. The identifier is 'SCIF-B'.  At version
+This is the format for match lists. The identifier is 'SCF-B'.  At version
 1.0, the following metadata tags are defined:
 
 Filtering: (optional)
@@ -205,6 +205,6 @@ then by end line.  Ranges within range sets should be sorted by natural
 order.  Range sets should be sorted by the natural order of their first
 member.
 
-It is strongly recommended that when a SCIF-B file is generated from
-SCIF-A files, the Normalization and Shred-Size headers should be copied
+It is strongly recommended that when a SCF-B file is generated from
+SCF-A files, the Normalization and Shred-Size headers should be copied
 into the result.
diff --git a/shred.h b/shred.h
index 1f5f946..5b32a38 100644
--- a/shred.h
+++ b/shred.h
@@ -30,33 +30,23 @@ struct sorthash_t
     char		*file;
 };
 
-struct scif_t
-{
-    char	*name;
-    FILE	*fp;
-    char	*normalization;
-    int		shred_size;
-    char	*hash_method;
-    char	*generator_program;
-    int		hashcount;
-};
-
 /* control bits, meant to be set at startup */
 extern int c_only;
 extern int rws;
 extern int debug;
 extern int shredsize;
 
+/* main.c functions */
+extern void report_time(char *legend, ...);
+extern void corehook(struct hash_t hash, const char *file);
+
 /* shredtree.c functions */
 extern char **sorted_file_list(const char *, int *);
 extern void shredfile(const char *, void (*hook)(struct hash_t, const char *));
-extern struct scif_t *init_merge(int argc, char *argv[]);
-extern struct sorthash_t *merge_hashes(struct scif_t *scif, int sciflen, 
-				       int *count);
 extern void sort_hashes(struct sorthash_t *hashlist, int hashcount);
-extern void emit_report(struct scif_t *, 
-			struct sorthash_t *obarray, int hashcount);
-extern void report_time(char *legend, ...);
 
+/* shredcompare.c functions */
+extern void merge_scf(const char *, FILE *);
+extern void emit_report(struct sorthash_t *obarray, int hashcount);
 
 /* shred.h ends here */
-- 
2.43.0