We can build for large files now.

diff --git a/Makefile b/Makefile
index 7aec63a1ea734dc6214ce5eb7ca2a88a67ec0460..3e08f8dcb15f362a9ce6350fbef94a58ac8f4b15 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,7 @@
 # Makefile for the comparator/filterator tools
+#
+# By building with -DLARGEFILES you can go to all 32-byte offsets.
+# This increases working-set size by 20% but handles > 65336 lines per file.
 
 VERS=1.5
 

diff --git a/comparator.xml b/comparator.xml
index 685bbc7ca4821f6d5aae2f3e778e7e5d978d5a5a..78caa730d084b45ae94304cded5b99e7139f193d 100644 (file)
--- a/comparator.xml
+++ b/comparator.xml
@@ -187,6 +187,10 @@ false-positive matches between shreds — 1 in
 by a meteor or spontaneously combust before this happens.)  There
 is no possibility of false negatives.</para>
 
+<para>This program handles a maximum of 65536 lines per file.  A full
+integer line-number range would increase working-set size by
+20%.</para>
+
 <para>You will get spurious results if you mix
 <filename>.scf</filename> files with different shred sizes or
 normalizations, or with settings for these that conflict with the

diff --git a/main.c b/main.c
index ad9dcc595bdb0d87f7c6f12f733ff2e849136c90..0dfe0055b5ebcf3ff0c44c88315fbb61061edc7c 100644 (file)
--- a/main.c
+++ b/main.c
@@ -99,7 +99,7 @@ static void write_scf(const char *tree, FILE *ofp)
 {
     char       **place, **list;
     int                file_count, progress, totalchunks;
-    u_int32_t  netfile_count, totallines = 0;
+    linecount_t        netfile_count, totallines = 0;
     char       buf[BUFSIZ];
 
     list = sorted_file_list(tree, &file_count);
@@ -123,7 +123,7 @@ static void write_scf(const char *tree, FILE *ofp)
     fputs("%%\n", ofp);
 
     netfile_count = htonl(file_count);
-    fwrite(&netfile_count, sizeof(u_int32_t), 1, ofp);
+    fwrite(&netfile_count, sizeof(linecount_t), 1, ofp);
     fprintf(stderr, "%% Reading %s...   ", tree);
     progress = totalchunks = 0;
     for (place = list; place < list + file_count; place++)
@@ -189,19 +189,19 @@ static void write_scf(const char *tree, FILE *ofp)
 
     /* the statistics trailer */
     totallines = htonl(totallines);
-    fwrite(&totallines, sizeof(u_int32_t), 1, ofp);
+    fwrite(&totallines, sizeof(linecount_t), 1, ofp);
 }
 
 static void read_scf(struct scf_t *scf)
 /* merge hashes from specified files into an in-code list */
 {
-    u_int32_t  filecount;
+    linecount_t        filecount;
     int hashcount = 0;
     struct stat sb;
 
     stat(scf->name, &sb);
     fprintf(stderr, "%% Reading hash list %s...   ", scf->name);
-    fread(&filecount, sizeof(u_int32_t), 1, scf->fp);
+    fread(&filecount, sizeof(linecount_t), 1, scf->fp);
     filecount = ntohl(filecount);
     while (filecount--)
     {
@@ -235,7 +235,7 @@ static void read_scf(struct scf_t *scf)
     }
     fprintf(stderr, "\b\b\b100%%...done, %d shreds\n", hashcount);
 
-    fread(&scf->totallines, sizeof(u_int32_t), 1, scf->fp);
+    fread(&scf->totallines, sizeof(linecount_t), 1, scf->fp);
 }
 
 static int merge_tree(char *tree)
@@ -243,7 +243,7 @@ static int merge_tree(char *tree)
 {
     char       **place, **list;
     int        old_entry_count, file_count, i;
-    u_int32_t  totallines = 0;
+    linecount_t        totallines = 0;
 
     old_entry_count = sort_count;
     file_count = 0;

diff --git a/shred.h b/shred.h
index 4da22a456654ace4954cc0d139b582d4b56b7808..6ae93b7d3f0cabc373051f33897811146cc455f8 100644 (file)
--- a/shred.h
+++ b/shred.h
@@ -3,16 +3,25 @@
 #include <sys/types.h>
 #include <netinet/in.h>
 
+#ifndef LARGEFILES
 /*
- * 65,536 lines should be enough, but maybe someday this will be 32 bits.
+ * 65,536 lines should be enough, but make it possible to compile with 32 bits.
  * The point of making it a short is that the data sets can get quite large,
  * so we want the hash_t and sorthash_t structures to be small.
  */
 typedef u_int16_t      linenum_t;
 #define TONET          htons
 #define FROMNET                ntohs
+#else
+typedef u_int32_t      linenum_t;
+#define TONET          htonl
+#define FROMNET                ntohl
+#endif
 #define UNIQUE_FLAG    (linenum_t)-1   /* 2s-complement assumption */
 
+/* use this to hold total line count of the entire source tree set */
+typedef u_int32_t      linecount_t;
+
 #define HASHSIZE       16
 struct hash_t
 {