From: Eric S. Raymond Date: Sun, 31 Aug 2003 06:09:27 +0000 (+0000) Subject: New packed file format. X-Git-Tag: 1.0~107 X-Git-Url: http://thyrsus.com/gitweb/?a=commitdiff_plain;h=ffe7665f83e4eed59ebbae64eaae8709cbab6b4c;p=comparator.git New packed file format. --- diff --git a/shredcompare.py b/shredcompare.py index 750d303..c9e0e09 100755 --- a/shredcompare.py +++ b/shredcompare.py @@ -2,7 +2,7 @@ # # shredcompare -- find common cliques in shred lists # -import os, sys, time, stat, getopt, bsddb +import os, sys, time, stat, getopt, bsddb, struct class SHIF: "SHIF-A file metadata container." @@ -81,22 +81,25 @@ hashcount = total = 0 def merge_hashes(fp, dict): "Read and merge hashes corresponding to one file." global hashcount - file = fp.readline().rstrip() - while True: - line = fp.readline() - if not line: - return False - if not line.strip(): - return True - (start, end, hashval) = line.split() + file = fp.readline() + if not file: + return False + file = file.strip() + # Record count comes right after the filename + (record_count,) = struct.unpack("!i", fp.read(4)) + while record_count > 0: + record_count -= 1 + # 4 bytes of start line number + 4 of end line number + 16 of MD5 hash + (start, end, hashval) = struct.unpack("!ii16s", fp.read(24)) if dict.has_key(hashval): oldval = dict[hashval] else: oldval = "" - dict[hashval] = oldval + file + "\t" + start + "\t" + end + "\n" + dict[hashval] = oldval + file + "\t" + `start` + "\t" + `end` + "\n" hashcount += 1 if hashcount % 10000 == 0: sys.stderr.write("%02.2f%% read.\n" % (fp.tell() / (total * 0.01))) + return True def item_factory(db): "Hide the ugliness that is sequential DB access in a generator." diff --git a/shredtree.c b/shredtree.c index f28c2ee..182f324 100644 --- a/shredtree.c +++ b/shredtree.c @@ -6,13 +6,16 @@ #include #include #include +#include +#include #include "md5.h" static int c_only = 0; static int rws = 0; static int debug = 0; static int shredsize = 5; -static int filecount; + +static int filecount, chunk_count; struct item { @@ -20,7 +23,20 @@ struct item struct item *next; } dummy; -struct item *head = &dummy; +static struct item *head = &dummy; + +/* + * This code depends on the assumption that this structure has + * no leading, trailing, or internal padding and takes up exactly + * 24 bytes of contiguous sequential memory. This should be the + * case on all modern word-addressible machines. + */ +struct hash_t +{ + u_int32_t start, end; + char hash[16]; +}; +static struct hash_t *outbuf; static int eligible(const char *file) /* is the specified file eligible to be compared? */ @@ -61,54 +77,42 @@ static void emit_chunk(shred *display, int linecount) { struct md5_ctx ctx; int i, firstline; - unsigned char hash[32], *cp; + unsigned char *cp; if (debug) { for (i = 0; i < shredsize; i++) if (display[i].line) { - putchar('\''); + fputc('\'', stderr); for (cp = display[i].line; *cp; cp++) if (*cp == '\n') - fputs("\\n", stdout); + fputs("\\n", stderr); else if (*cp == '\t') - fputs("\\t", stdout); + fputs("\\t", stderr); else - putchar(*cp); - putchar('\''); - putchar('\n'); + fputc(*cp, stderr); + fputc('\'', stderr); + fputc('\n', stderr); } } - /* flush completed chunk */ + outbuf = (struct hash_t *)realloc(outbuf, + sizeof(struct hash_t) * (chunk_count+1)); + + /* build completed chunk onto end of array */ md5_init_ctx(&ctx); for (i = 0; i < shredsize; i++) if (display[i].line) md5_process_bytes(display[i].line, strlen(display[i].line), &ctx); - md5_finish_ctx(&ctx, (void *)hash); + md5_finish_ctx(&ctx, (void *)&outbuf[chunk_count].hash); for (i = shredsize - 1; i >= 0; i--) if (display[i].line) firstline = i; - fprintf(stdout, - "%d\t%d\t%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n", - display[firstline].number, linecount, - hash[0], - hash[1], - hash[2], - hash[3], - hash[4], - hash[5], - hash[6], - hash[7], - hash[8], - hash[9], - hash[10], - hash[11], - hash[12], - hash[13], - hash[14], - hash[15]); + firstline = display[firstline].number; + outbuf[chunk_count].start = htonl(firstline); + outbuf[chunk_count].end = htonl(linecount); + chunk_count++; } void shredfile(const char *file) @@ -116,7 +120,7 @@ void shredfile(const char *file) { FILE *fp; char buf[BUFSIZ]; - int i, linecount, accepted; + int i, linecount, accepted, net_chunks; shred *display; if ((fp = fopen(file, "r")) == NULL) @@ -127,10 +131,9 @@ void shredfile(const char *file) } display = (shred *)calloc(sizeof(shred), shredsize); + outbuf = (struct hash_t *)calloc(sizeof(struct hash_t), 1); - puts(file); - - accepted = linecount = 0; + accepted = linecount = chunk_count = 0; while(fgets(buf, sizeof(buf), fp) != NULL) { linecount++; @@ -154,9 +157,15 @@ void shredfile(const char *file) } if (linecount < shredsize) emit_chunk(display, linecount); + net_chunks = htonl(chunk_count); + + /* the actual output */ + puts(file); + fwrite((char *)&net_chunks, sizeof(u_int32_t), 1, stdout); + fwrite(outbuf, sizeof(struct hash_t), chunk_count, stdout); - puts(""); free(display); + free(outbuf); fclose(fp); } diff --git a/shredtree.py b/shredtree.py index 453ac2f..56b423e 100755 --- a/shredtree.py +++ b/shredtree.py @@ -2,7 +2,7 @@ # # shredtree -- generate hash list in SHIF-A format for a given source tree -import sys, os, os.path, re, md5, getopt +import sys, os, os.path, re, md5, getopt, struct whitespace = re.compile(r"[ \t\n]+") @@ -23,21 +23,23 @@ def normalize(line): else: return line -def emit_chunk(display, linecount): - m = md5.new() +def emit_chunk(display, file, linecount): + startline = display[0][1] + if debug: + sys.stderr.write("%% %s:%d-%d\n" % (file, startline, linecount)) + m = md5.new() for line in display: if debug: - print `line[0]` + sys.stderr.write(`line[0]` + "\n") m.update(line[0]) - print "%d\t%d\t%s" % (display[0][1], linecount, m.hexdigest()) + return struct.pack("!ii", startline, linecount) + m.digest() def shredfile(file): "Emit shred tuples corresponding to a specified file." - print file fp = open(file, "r") display = [] - - accepted = linecount = 0; + outdata = "" + chunk_count = accepted = linecount = 0; while True: line = fp.readline() if not line: @@ -53,7 +55,8 @@ def shredfile(file): # flush completed chunk if accepted >= shredsize: - emit_chunk(display, linecount) + outdata += emit_chunk(display, file, linecount) + chunk_count += 1 # shreds in progress are shifted down */ if len(display) >= shredsize: @@ -61,8 +64,10 @@ def shredfile(file): fp.close() if linecount < shredsize: - emit_chunk(display, linecount) - print "" + outdata += emit_chunk(display, file, linecount) + chunk_count += 1 + # Write the actual data + sys.stdout.write(file + "\n" + struct.pack("!i", chunk_count) + outdata) if __name__ == '__main__': try: @@ -91,7 +96,7 @@ if __name__ == '__main__': shredsize = int(val) elif opt == '-w': ws = True - elif opt == '-d': + elif opt == '-x': debug = True if args: print "#SHIF-A 1.0"