I think the hash stuff works now.

diff --git a/shredcompare.py b/shredcompare.py
index bec0f26d2a10b7f8e604caf898f86afac77d79ce..d4337d23667e949061fab4469c68a775f8a20e05 100755 (executable)
--- a/shredcompare.py
+++ b/shredcompare.py
@@ -96,24 +96,22 @@ def shredtree(tree):
     shreds = {}
     if verbose:
         print "% Shredding", tree
-    if os.path.exists(tree + ".hash"):
-        # There's a precomputed shred list.
-        rfp = open(tree + ".hash")
-        shreds = cPickle.load(rfp)
-        rfp.close()
-    else:
-        # No precomputed hashes.  Prepare a news shred list.
-        for root, dirs, files in os.walk(tree):
-            for file in files:
-                path = os.path.join(root, file)
-                if eligible(path):
-                    shreds.update(shredfile(path, tree))
+    # No precomputed hashes.  Prepare a news shred list.
+    for root, dirs, files in os.walk(tree):
+        for file in files:
+            path = os.path.join(root, file)
+            if eligible(path):
+                shreds.update(shredfile(path, tree))
     return shreds
 
 def shredcompare(tree1, tree2):
-    "Compare two trees.  Returns a list of matching Shred pairs"
-    shreds1 = shredtree(tree1)
-    shreds2 = shredtree(tree2)
+    "Compare two trees.  Returns a list of matching shred pairs"
+    shreds1 = read_hash(tree1)
+    if not shreds1:
+        shreds1 = shredtree(tree1)
+    shreds2 = read_hash(tree2)
+    if not shreds2:
+        shreds2 = shredtree(tree2)
     # Nuke everything but checksum matches between the trees
     for key in shreds1.keys():
         if not key in shreds2:
@@ -126,10 +124,10 @@ def shredcompare(tree1, tree2):
     for key in shreds1.keys():
         matches.append((shreds1[key], shreds2[key]))
     # Merge overlapping ranges if possible.
-    # This in O(n**2) in the number of matching chunks.
+    # This is O(n**2) in the number of matching chunks.
     # Should performance ever become a problem, break this up into
-    # separate passes over the chunks in each file. (Corresponds
-    # to eliminating a bunch of off-diagonal entries.
+    # separate passes over the chunks in each file. (Would correspond
+    # to eliminating a bunch of off-diagonal entries.)
     retry = True
     while retry:
         retry = False
@@ -155,6 +153,32 @@ def shredcompare(tree1, tree2):
                 retry = True
     return filter(lambda x: x, matches)
 
+def read_hash(tree):
+    "Try to read a hash digest corresponding to a tree."
+    if not os.path.exists(tree + ".hash"):
+        return None
+    else:
+        # There's a precomputed shred list.
+        rfp = open(tree + ".hash")
+        shreds = {}
+        while True:
+            line = rfp.readline()
+            if not line:
+                break
+            digest = line[:16]
+            (file, range) = line[16:].split(":")
+            (start, end) = range.split("-")
+            shreds[digest] = Shred(file, int(start), int(end))
+        rfp.close()
+        return shreds
+
+def write_hash(shreds, tree):
+    "Write a hash digest corresponding to a tree."
+    wfp = open(tree + ".hash", "w")
+    for key in shreds:
+        wfp.write(key + str(shreds[key]) + "\n")
+    wfp.close()
+
 def report_time(legend=None):
     "Report time since start_time was set."
     global mark_time
@@ -184,11 +208,9 @@ if __name__ == '__main__':
     report_time()
     if makehash:
         shreds = shredtree(makehash)
-        wfp = open(makehash + ".hash", "w")
         report_time("Shred list complete.")
-        cPickle.dump(shreds, wfp)
-        wfp.close()
-        report_time("Pickling complete")
+        write_hash(shreds, makehash)
+        report_time("Digesting complete")
     else:
         matches = shredcompare(args[0], args[1])
         report_time("Match list complete")