#
# shredcompare -- find common cliques in shred lists
#
-import os, sys, time, stat, getopt, bsddb
+import os, sys, time, stat, getopt, bsddb, struct
class SHIF:
"SHIF-A file metadata container."
def merge_hashes(fp, dict):
"Read and merge hashes corresponding to one file."
global hashcount
- file = fp.readline().rstrip()
- while True:
- line = fp.readline()
- if not line:
- return False
- if not line.strip():
- return True
- (start, end, hashval) = line.split()
+ file = fp.readline()
+ if not file:
+ return False
+ file = file.strip()
+ # Record count comes right after the filename
+ (record_count,) = struct.unpack("!i", fp.read(4))
+ while record_count > 0:
+ record_count -= 1
+ # 4 bytes of start line number + 4 of end line number + 16 of MD5 hash
+ (start, end, hashval) = struct.unpack("!ii16s", fp.read(24))
if dict.has_key(hashval):
oldval = dict[hashval]
else:
oldval = ""
- dict[hashval] = oldval + file + "\t" + start + "\t" + end + "\n"
+ dict[hashval] = oldval + file + "\t" + `start` + "\t" + `end` + "\n"
hashcount += 1
if hashcount % 10000 == 0:
sys.stderr.write("%02.2f%% read.\n" % (fp.tell() / (total * 0.01)))
+ return True
def item_factory(db):
"Hide the ugliness that is sequential DB access in a generator."
#include <ftw.h>
#include <errno.h>
#include <alloca.h>
+#include <netinet/in.h>
+#include <sys/types.h>
#include "md5.h"
static int c_only = 0;
static int rws = 0;
static int debug = 0;
static int shredsize = 5;
-static int filecount;
+
+static int filecount, chunk_count;
struct item
{
struct item *next;
}
dummy;
-struct item *head = &dummy;
+static struct item *head = &dummy;
+
+/*
+ * This code depends on the assumption that this structure has
+ * no leading, trailing, or internal padding and takes up exactly
+ * 24 bytes of contiguous sequential memory. This should be the
+ * case on all modern word-addressible machines.
+ */
+struct hash_t
+{
+ u_int32_t start, end;
+ char hash[16];
+};
+static struct hash_t *outbuf;
static int eligible(const char *file)
/* is the specified file eligible to be compared? */
{
struct md5_ctx ctx;
int i, firstline;
- unsigned char hash[32], *cp;
+ unsigned char *cp;
if (debug)
{
for (i = 0; i < shredsize; i++)
if (display[i].line)
{
- putchar('\'');
+ fputc('\'', stderr);
for (cp = display[i].line; *cp; cp++)
if (*cp == '\n')
- fputs("\\n", stdout);
+ fputs("\\n", stderr);
else if (*cp == '\t')
- fputs("\\t", stdout);
+ fputs("\\t", stderr);
else
- putchar(*cp);
- putchar('\'');
- putchar('\n');
+ fputc(*cp, stderr);
+ fputc('\'', stderr);
+ fputc('\n', stderr);
}
}
- /* flush completed chunk */
+ outbuf = (struct hash_t *)realloc(outbuf,
+ sizeof(struct hash_t) * (chunk_count+1));
+
+ /* build completed chunk onto end of array */
md5_init_ctx(&ctx);
for (i = 0; i < shredsize; i++)
if (display[i].line)
md5_process_bytes(display[i].line, strlen(display[i].line), &ctx);
- md5_finish_ctx(&ctx, (void *)hash);
+ md5_finish_ctx(&ctx, (void *)&outbuf[chunk_count].hash);
for (i = shredsize - 1; i >= 0; i--)
if (display[i].line)
firstline = i;
- fprintf(stdout,
- "%d\t%d\t%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
- display[firstline].number, linecount,
- hash[0],
- hash[1],
- hash[2],
- hash[3],
- hash[4],
- hash[5],
- hash[6],
- hash[7],
- hash[8],
- hash[9],
- hash[10],
- hash[11],
- hash[12],
- hash[13],
- hash[14],
- hash[15]);
+ firstline = display[firstline].number;
+ outbuf[chunk_count].start = htonl(firstline);
+ outbuf[chunk_count].end = htonl(linecount);
+ chunk_count++;
}
void shredfile(const char *file)
{
FILE *fp;
char buf[BUFSIZ];
- int i, linecount, accepted;
+ int i, linecount, accepted, net_chunks;
shred *display;
if ((fp = fopen(file, "r")) == NULL)
}
display = (shred *)calloc(sizeof(shred), shredsize);
+ outbuf = (struct hash_t *)calloc(sizeof(struct hash_t), 1);
- puts(file);
-
- accepted = linecount = 0;
+ accepted = linecount = chunk_count = 0;
while(fgets(buf, sizeof(buf), fp) != NULL)
{
linecount++;
}
if (linecount < shredsize)
emit_chunk(display, linecount);
+ net_chunks = htonl(chunk_count);
+
+ /* the actual output */
+ puts(file);
+ fwrite((char *)&net_chunks, sizeof(u_int32_t), 1, stdout);
+ fwrite(outbuf, sizeof(struct hash_t), chunk_count, stdout);
- puts("");
free(display);
+ free(outbuf);
fclose(fp);
}
#
# shredtree -- generate hash list in SHIF-A format for a given source tree
-import sys, os, os.path, re, md5, getopt
+import sys, os, os.path, re, md5, getopt, struct
whitespace = re.compile(r"[ \t\n]+")
else:
return line
-def emit_chunk(display, linecount):
- m = md5.new()
+def emit_chunk(display, file, linecount):
+ startline = display[0][1]
+ if debug:
+ sys.stderr.write("%% %s:%d-%d\n" % (file, startline, linecount))
+ m = md5.new()
for line in display:
if debug:
- print `line[0]`
+ sys.stderr.write(`line[0]` + "\n")
m.update(line[0])
- print "%d\t%d\t%s" % (display[0][1], linecount, m.hexdigest())
+ return struct.pack("!ii", startline, linecount) + m.digest()
def shredfile(file):
"Emit shred tuples corresponding to a specified file."
- print file
fp = open(file, "r")
display = []
-
- accepted = linecount = 0;
+ outdata = ""
+ chunk_count = accepted = linecount = 0;
while True:
line = fp.readline()
if not line:
# flush completed chunk
if accepted >= shredsize:
- emit_chunk(display, linecount)
+ outdata += emit_chunk(display, file, linecount)
+ chunk_count += 1
# shreds in progress are shifted down */
if len(display) >= shredsize:
fp.close()
if linecount < shredsize:
- emit_chunk(display, linecount)
- print ""
+ outdata += emit_chunk(display, file, linecount)
+ chunk_count += 1
+ # Write the actual data
+ sys.stdout.write(file + "\n" + struct.pack("!i", chunk_count) + outdata)
if __name__ == '__main__':
try:
shredsize = int(val)
elif opt == '-w':
ws = True
- elif opt == '-d':
+ elif opt == '-x':
debug = True
if args:
print "#SHIF-A 1.0"
projects / comparator.git / commitdiff