#include <stdio.h>
#include <unistd.h>
#include <stdarg.h>
+#include <string.h>
#include "shred.h"
+struct scf_t
+{
+ char *name;
+ FILE *fp;
+ char *normalization;
+ int shred_size;
+ char *hash_method;
+ char *generator_program;
+ struct scf_t *next;
+};
+static struct scf_t dummy_scf, *scf_head = &dummy_scf;
+
static int chunk_count, sort_count;
static struct hash_t *chunk_buffer;
static struct sorthash_t *sort_buffer;
+static int is_scf_file(const char *file)
+/* is the specified file an SCF hash list? */
+{
+ char buf[10];
+ FILE *fp = fopen(file, "r");
+
+ if (!fgets(buf, sizeof(buf), fp))
+ {
+ fclose(fp);
+ return(0);
+ }
+ fclose(fp);
+ return(!strncmp(buf, "#SCF-A ", 7));
+}
+
/*
* We differentiate these two functions in order to avaid the memory
* overhead of the extra (char *) when assembling shred lists to be written
}
-static void corehook(struct hash_t hash, const char *file)
+void corehook(struct hash_t hash, const char *file)
/* hook to store hash and file */
{
sort_buffer = (struct sorthash_t *)realloc(sort_buffer,
list = sorted_file_list(tree, &file_count);
- fputs("#SCIF-A 1.0\n", ofp);
+ fputs("#SCF-A 1.0\n", ofp);
fputs("Generator-Program: comparator 1.0\n", ofp);
fputs("Hash-Method: MD5\n", ofp);
fprintf(ofp, "Normalization: %s\n", rws ? "remove_whitespace" : "none");
}
}
-static void generate_shredlist(int argc, char *argv[])
-/* generate an in-core list of sorthash structures */
+static void merge_tree(char *tree)
+/* add to the in-core list of sorthash structures from a tree */
+{
+ char **place, **list;
+ int oldcount, file_count;
+
+ oldcount = sort_count;
+ fprintf(stderr, "%% Reading %s...", tree);
+ list = sorted_file_list(tree, &file_count);
+ for (place = list; place < list + file_count; place++)
+ shredfile(*place, corehook);
+ fprintf(stderr, "%d entries\n", sort_count - oldcount);
+ free(list);
+}
+
+static void init_scf(char *file)
+/* add to the in-core list of sorthash structures from a SCF file */
{
- int i, file_count;
+ struct scf_t *new;
+ char buf[BUFSIZ];
- sort_buffer = (struct sorthash_t*)calloc(sizeof(struct sorthash_t),1);
- sort_count = 0;
+ new = (struct scf_t *)malloc(sizeof(struct scf_t));
- for (i = 0; i < argc; i++)
+ /* read in the SCF metadata block and add it to the in-core list */
+ new->name = strdup(file);
+ new->fp = fopen(new->name, "r");
+ fgets(buf, sizeof(buf), new->fp);
+ if (strncmp(buf, "#SCF-A ", 8))
{
- char **place, **list;
- int oldcount;
-
- oldcount = sort_count;
- fprintf(stderr, "%% Reading %s...", argv[i]);
- list = sorted_file_list(argv[i], &file_count);
- for (place = list; place < list + file_count; place++)
- shredfile(*place, corehook);
- fprintf(stderr, "%d entries\n", sort_count - oldcount);
- free(list);
+ fprintf(stderr,
+ "shredcompare: %s is not a SCF-A file.",
+ new->name);
+ exit(1);
}
+ while (fgets(buf, sizeof(buf), new->fp) != NULL)
+ {
+ char *value;
+
+ if (!strcmp(buf, "%%\n"))
+ break;
+ value = strchr(buf, ':');
+ *value++ = '\0';
+ while(*value == ' ')
+ value++;
+ strchr(value, '\n')[0] = '\0';
+
+ if (!strcmp(buf, "Normalization"))
+ new->normalization = strdup(value);
+ else if (!strcmp(buf, "Shred-Size"))
+ new->shred_size = atoi(value);
+ else if (!strcmp(buf, "Hash-Method"))
+ new->hash_method = strdup(value);
+ else if (!strcmp(buf, "Generator-Program"))
+ new->generator_program = strdup(value);
+ }
+
+ new->name = strdup(file);
+ new->next = scf_head;
+ scf_head = new;
}
void report_time(char *legend, ...)
+/* report on time since last report_mark */
{
static time_t mark_time;
time_t endtime = time(NULL);
extern int optind; /* set by getopt */
int status, file_only, compile_only;
+ struct scf_t *scf;
compile_only = file_only = 0;
while ((status = getopt(argc, argv, "cd:hs:w")) != EOF)
case 'h':
default:
fprintf(stderr,"usage: shredtree [-c] [-d dir ] [-h] [-s shredsize] [-w] [-x] path\n");
- fprintf(stderr," -c = generate SCIF files\n");
+ fprintf(stderr," -c = generate SCF files\n");
fprintf(stderr," -d dir = change directory before digesting.\n");
fprintf(stderr," -h = help (display this message).\n");
fprintf(stderr," -s size = set shred size (default %d)\n",
}
}
- if (compile_only)
- generate_shredfile(argv[optind], stdout);
- else
+ if (!compile_only)
{
- struct scif_t localblock;
+ sort_buffer = (struct sorthash_t*)calloc(sizeof(struct sorthash_t),1);
+ sort_count = 0;
+ }
- localblock.hash_method = "MD5";
- localblock.normalization = rws ? "remove_whitespace" : "none";
- localblock.shred_size = shredsize;
- localblock.generator_program = "comparator 1.0";
+ report_time(NULL);
+ for(; optind < argc; optind++)
+ {
+ char *source = argv[optind];
- report_time(NULL);
- generate_shredlist(argc-optind, argv+optind);
- report_time("Hash merge done, %d entries", sort_count);
- sort_hashes(sort_buffer, sort_count);
- report_time("Sort done");
- emit_report(&localblock, sort_buffer, sort_count);
+ if (is_scf_file(source))
+ init_scf(source);
+ else if (compile_only)
+ {
+ char *outfile = alloca(strlen(source) + 4);
+ FILE *ofp;
+
+ strcpy(outfile, source);
+ strcat(outfile, ".scf");
+ ofp = fopen(outfile, "w");
+ if (!ofp)
+ {
+ fprintf(stderr, "comparator: couldn't open output file %s\n",
+ outfile);
+ exit(1);
+ }
+ generate_shredfile(source, ofp);
+ fclose(ofp);
+ }
+ else
+ merge_tree(source);
}
-#if 0
+ /* if there were no SCFs, create a dummy one */
+ if (scf_head == &dummy_scf)
+ {
+ dummy_scf.hash_method = "MD5";
+ dummy_scf.normalization = rws ? "remove_whitespace" : "none";
+ dummy_scf.shred_size = shredsize;
+ dummy_scf.generator_program = "comparator 1.0";
+ }
else
{
- int hashcount;
- struct sorthash_t *obarray;
- struct scif_t *sciflist;
-
- report_time(NULL);
- sciflist = init_merge(argc-optind, argv+optind);
- obarray = merge_hashes(sciflist, argc-optind, &hashcount);
- report_time("Hash merge done, %d entries", hashcount);
- sort_hashes(obarray, hashcount);
+ /* consistency checks on the SCFs */
+ for (scf = scf_head; scf->next; scf = scf->next)
+ {
+ struct scf_t *next = scf + 1;
+
+ if (strcmp(scf->normalization, next->normalization))
+ {
+ fprintf(stderr,
+ "shredcompare: normalizations of %s and %s don't match\n",
+ scf->name, next->name);
+ exit(1);
+ }
+ else if (scf->shred_size != next->shred_size)
+ {
+ fprintf(stderr,
+ "shredcompare: shred sizes of %s and %s don't match\n",
+ scf->name, next->name);
+ exit(1);
+
+ }
+ else if (strcmp(scf->hash_method, next->hash_method))
+ {
+ fprintf(stderr,
+ "shredcompare: hash methods of %s and %s don't match\n",
+ scf->name, next->name);
+ exit(1);
+ }
+ }
+
+ /* finish reading in all SCFs */
+ for (scf = scf_head; scf->next; scf = scf->next)
+ {
+ merge_scf(scf->name, scf->fp);
+ fclose(scf->fp);
+ }
+
+ }
+
+ if (!compile_only)
+ {
+ /* now we're ready to emit the report */
+ puts("#SCF-B 1.0");
+ printf("Hash-Method: %s\n", scf_head->hash_method);
+ puts("Merge-Program: comparator 1.0");
+ printf("Normalization: %s\n", scf_head->normalization);
+ printf("Shred-Size: %d\n", scf_head->shred_size);
+ puts("%%");
+
+ report_time("Hash merge done, %d entries", sort_count);
+ sort_hashes(sort_buffer, sort_count);
report_time("Sort done");
- emit_report(sciflist, obarray, hashcount);
+
+ emit_report(sort_buffer, sort_count);
}
-#endif /* FOO */
exit(0);
}
dummy_item;
static struct item *head = &dummy_item;
-static struct scif_t *sciflist;
+static struct scf_t *scflist;
struct range_t
{
dummy_match;
static struct match_t *reduced = &dummy_match;
-static void file_intern(const char *file)
-/* stash the name of a file */
-{
- struct item *new;
-
- new = (struct item *)malloc(sizeof(struct item));
- new->file = strdup(file);
- new->next = head;
- head = new;
-}
-
-struct scif_t *init_merge(int argc, char *argv[])
-{
- struct scif_t *sp;
-
- /* set up metadata blocks for the hash files */
- sciflist = (struct scif_t *)calloc(sizeof(struct scif_t), argc);
- for (sp = sciflist; sp < sciflist + argc; sp++)
- {
- char buf[BUFSIZ];
-
- sp->name = strdup(argv[sp - sciflist]);
- sp->fp = fopen(sp->name, "r");
- fgets(buf, sizeof(buf), sp->fp);
- if (strncmp(buf, "#SCIF-A ", 8))
- {
- fprintf(stderr,
- "shredcompare: %s is not a SCIF-A file.",
- sp->name);
- exit(1);
- }
- while (fgets(buf, sizeof(buf), sp->fp) != NULL)
- {
- char *value;
-
- if (!strcmp(buf, "%%\n"))
- break;
- value = strchr(buf, ':');
- *value++ = '\0';
- while(*value == ' ')
- value++;
- strchr(value, '\n')[0] = '\0';
-
- if (!strcmp(buf, "Normalization"))
- sp->normalization = strdup(value);
- else if (!strcmp(buf, "Shred-Size"))
- sp->shred_size = atoi(value);
- else if (!strcmp(buf, "Hash-Method"))
- sp->hash_method = strdup(value);
- else if (!strcmp(buf, "Generator-Program"))
- sp->generator_program = strdup(value);
- }
- }
-
- /* consistency checks */
- for (sp = sciflist; sp < sciflist + argc-1; sp++)
- {
- struct scif_t *next = sp + 1;
-
- if (strcmp(sp->normalization, next->normalization))
- {
- fprintf(stderr,
- "shredcompare: normalizations of %s and %s don't match\n",
- sp->name, next->name);
- exit(1);
- }
- else if (sp->shred_size != next->shred_size)
- {
- fprintf(stderr,
- "shredcompare: shred sizes of %s and %s don't match\n",
- sp->name, next->name);
- exit(1);
-
- }
- else if (strcmp(sp->hash_method, next->hash_method))
- {
- fprintf(stderr,
- "shredcompare: hash methods of %s and %s don't match\n",
- sp->name, next->name);
- exit(1);
- }
- }
-
- return(sciflist);
-}
-
-struct sorthash_t *merge_hashes(struct scif_t *sciflist, int sciflen, int *count)
+void merge_scf(const char *name, FILE *fp)
/* merge hashes from specified files into an in-code list */
{
- struct scif_t *sp;
- long total;
- struct sorthash_t *obarray;
- int hashcount;
- struct stat sb;
-
- /* compute total data to be read, we'll use this for the progress meter */
- total = 0;
- for (sp = sciflist; sp < sciflist + sciflen; sp++)
- {
- stat(sp->name, &sb);
- total += sb.st_size - ftell(sp->fp);
- }
- obarray=(struct sorthash_t *)calloc(sizeof(struct sorthash_t),
- total/sizeof(struct hash_t));
- if (!obarray)
- {
- fprintf(stderr, "shredcompare: insufficient memory\n");
- exit(1);
- }
+ u_int32_t sectcount;
+ int hashcount = 0;
- /* read in all hashes */
- hashcount = 0;
- for (sp = sciflist; sp < sciflist + sciflen; sp++)
+ fprintf(stderr, "%% Reading %s... ", name);
+ fread(§count, sizeof(u_int32_t), 1, fp);
+ sectcount = ntohl(sectcount);
+ while (sectcount--)
{
- u_int32_t sectcount;
-
- fprintf(stderr, "Reading %s... ", sp->name);
- fread(§count, sizeof(u_int32_t), 1, sp->fp);
- sectcount = ntohl(sectcount);
- while (sectcount--)
+ char buf[BUFSIZ];
+ linenum_t chunks;
+ struct item *new;
+
+ fgets(buf, sizeof(buf), fp);
+ *strchr(buf, '\n') = '\0';
+ new = (struct item *)malloc(sizeof(struct item));
+ new->file = strdup(name);
+ new->next = head;
+ head = new;
+ fread(&chunks, sizeof(linenum_t), 1, fp);
+ chunks = FROMNET(chunks);
+
+ while (chunks--)
{
- char buf[BUFSIZ];
- linenum_t chunks;
-
- fgets(buf, sizeof(buf), sp->fp);
- *strchr(buf, '\n') = '\0';
- file_intern(buf); /* real work done here */
- fread(&chunks, sizeof(linenum_t), 1, sp->fp);
- chunks = FROMNET(chunks);
-
- while (chunks--)
- {
- struct hash_t this;
-
- fread(&this, sizeof(struct hash_t), 1, sp->fp);
- this.start = FROMNET(this.start);
- this.end = FROMNET(this.end);
- obarray[hashcount].file = head->file;
- obarray[hashcount].hash = this;
- hashcount++;
- sp->hashcount++;
- if (hashcount % 10000 == 0)
- fprintf(stderr, "\b\b\b%02.0f%%", (ftell(sp->fp) / (total * 0.01)));
- }
+ struct hash_t this;
+
+ fread(&this.hash, sizeof(struct hash_t), 1, fp);
+ this.start = FROMNET(this.start);
+ this.end = FROMNET(this.end);
+ corehook(this, head->file);
+ hashcount++;
+#ifdef FIXME
+ if (hashcount % 10000 == 0)
+ fprintf(stderr,"\b\b\b%02.0f%%",(ftell(fp) / (total * 0.01)));
+#endif
}
- fprintf(stderr, "\b\b\b100%%...done, %d entries\n", sp->hashcount);
}
-
- *count = hashcount;
- return obarray;
+ fprintf(stderr, "\b\b\b100%%...done, %d entries\n", hashcount);
}
static int merge_ranges(struct range_t *p, struct range_t *q, int nmatches)
return(0);
}
-void emit_report(struct scif_t *scif,
- struct sorthash_t *obarray, int hashcount)
+void emit_report(struct sorthash_t *obarray, int hashcount)
{
struct match_t *hitlist, *sorted, *match;
int i, matchcount;
#ifdef ODEBUG
+ struct sorthash_t *np;
+
for (np = obarray; np < obarray + hashcount; np++)
printf("%d: %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x %s:%d:%d\n",
np-obarray,
hitlist = reduce_matches(obarray, hashcount);
report_time("Reduction done");
- puts("#SCIF-B 1.0");
- printf("Hash-Method: %s\n", scif->hash_method);
- puts("Merge-Program: comparator 1.0");
- printf("Normalization: %s\n", scif->normalization);
- printf("Shred-Size: %d\n", scif->shred_size);
- puts("%%");
-
/* we go through a little extra effort to emit a sorted list */
matchcount = 0;
for (match = hitlist; match->next; match = match->next)
printf("%s:%d:%d\n", rp->file, rp->start, rp->end);
}
- printf("-\n");
+ printf("%%%%\n");
}
}
-$Id: interchange,v 1.1 2003/09/03 03:04:51 esr Exp esr $
+$Id: interchange,v 1.2 2003/09/03 03:43:41 esr Exp esr $
As of September 2003 there are several individuals and research groups engaged
in building tools for doing analysis of similarities among source-code trees.
able to describe what reduction and filtering steps were performed.
The remainder of this document describes file formats to fill these needs.
-These formats will be known as SCIF (source Comparison Interchange Format)
+These formats will be known as SCF (source Comparison Interchange Format)
files, pronounced "skiff".
Each format consists of three sections: an identification line,
The ordering rules give every data-set a canonical form so that
operations like diff(1) or comm(1) can give meaningful results.
-SCIF-A:
+SCF-A:
-This is the format for hash lists. The identifier is 'SCIF-A'. At version
+This is the format for hash lists. The identifier is 'SCF-A'. At version
1.0, the following metadata tags are defined:
Normalization: (mandatory)
fast as possible, because shred data sets are quite large in holding
down I/O is an important consideration.
-SCIF-B:
+SCF-B:
-This is the format for match lists. The identifier is 'SCIF-B'. At version
+This is the format for match lists. The identifier is 'SCF-B'. At version
1.0, the following metadata tags are defined:
Filtering: (optional)
order. Range sets should be sorted by the natural order of their first
member.
-It is strongly recommended that when a SCIF-B file is generated from
-SCIF-A files, the Normalization and Shred-Size headers should be copied
+It is strongly recommended that when a SCF-B file is generated from
+SCF-A files, the Normalization and Shred-Size headers should be copied
into the result.
char *file;
};
-struct scif_t
-{
- char *name;
- FILE *fp;
- char *normalization;
- int shred_size;
- char *hash_method;
- char *generator_program;
- int hashcount;
-};
-
/* control bits, meant to be set at startup */
extern int c_only;
extern int rws;
extern int debug;
extern int shredsize;
+/* main.c functions */
+extern void report_time(char *legend, ...);
+extern void corehook(struct hash_t hash, const char *file);
+
/* shredtree.c functions */
extern char **sorted_file_list(const char *, int *);
extern void shredfile(const char *, void (*hook)(struct hash_t, const char *));
-extern struct scif_t *init_merge(int argc, char *argv[]);
-extern struct sorthash_t *merge_hashes(struct scif_t *scif, int sciflen,
- int *count);
extern void sort_hashes(struct sorthash_t *hashlist, int hashcount);
-extern void emit_report(struct scif_t *,
- struct sorthash_t *obarray, int hashcount);
-extern void report_time(char *legend, ...);
+/* shredcompare.c functions */
+extern void merge_scf(const char *, FILE *);
+extern void emit_report(struct sorthash_t *obarray, int hashcount);
/* shred.h ends here */
projects / comparator.git / commitdiff