From 5bd0bffad6fad67bbadbc6d5215ebfe71139023a Mon Sep 17 00:00:00 2001 From: "Eric S. Raymond" Date: Sat, 13 Sep 2003 07:36:14 +0000 Subject: [PATCH] We can build for large files now. --- Makefile | 3 +++ comparator.xml | 4 ++++ main.c | 14 +++++++------- shred.h | 11 ++++++++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 7aec63a..3e08f8d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ # Makefile for the comparator/filterator tools +# +# By building with -DLARGEFILES you can go to all 32-byte offsets. +# This increases working-set size by 20% but handles > 65336 lines per file. VERS=1.5 diff --git a/comparator.xml b/comparator.xml index 685bbc7..78caa73 100644 --- a/comparator.xml +++ b/comparator.xml @@ -187,6 +187,10 @@ false-positive matches between shreds — 1 in by a meteor or spontaneously combust before this happens.) There is no possibility of false negatives. +This program handles a maximum of 65536 lines per file. A full +integer line-number range would increase working-set size by +20%. + You will get spurious results if you mix .scf files with different shred sizes or normalizations, or with settings for these that conflict with the diff --git a/main.c b/main.c index ad9dcc5..0dfe005 100644 --- a/main.c +++ b/main.c @@ -99,7 +99,7 @@ static void write_scf(const char *tree, FILE *ofp) { char **place, **list; int file_count, progress, totalchunks; - u_int32_t netfile_count, totallines = 0; + linecount_t netfile_count, totallines = 0; char buf[BUFSIZ]; list = sorted_file_list(tree, &file_count); @@ -123,7 +123,7 @@ static void write_scf(const char *tree, FILE *ofp) fputs("%%\n", ofp); netfile_count = htonl(file_count); - fwrite(&netfile_count, sizeof(u_int32_t), 1, ofp); + fwrite(&netfile_count, sizeof(linecount_t), 1, ofp); fprintf(stderr, "%% Reading %s... ", tree); progress = totalchunks = 0; for (place = list; place < list + file_count; place++) @@ -189,19 +189,19 @@ static void write_scf(const char *tree, FILE *ofp) /* the statistics trailer */ totallines = htonl(totallines); - fwrite(&totallines, sizeof(u_int32_t), 1, ofp); + fwrite(&totallines, sizeof(linecount_t), 1, ofp); } static void read_scf(struct scf_t *scf) /* merge hashes from specified files into an in-code list */ { - u_int32_t filecount; + linecount_t filecount; int hashcount = 0; struct stat sb; stat(scf->name, &sb); fprintf(stderr, "%% Reading hash list %s... ", scf->name); - fread(&filecount, sizeof(u_int32_t), 1, scf->fp); + fread(&filecount, sizeof(linecount_t), 1, scf->fp); filecount = ntohl(filecount); while (filecount--) { @@ -235,7 +235,7 @@ static void read_scf(struct scf_t *scf) } fprintf(stderr, "\b\b\b100%%...done, %d shreds\n", hashcount); - fread(&scf->totallines, sizeof(u_int32_t), 1, scf->fp); + fread(&scf->totallines, sizeof(linecount_t), 1, scf->fp); } static int merge_tree(char *tree) @@ -243,7 +243,7 @@ static int merge_tree(char *tree) { char **place, **list; int old_entry_count, file_count, i; - u_int32_t totallines = 0; + linecount_t totallines = 0; old_entry_count = sort_count; file_count = 0; diff --git a/shred.h b/shred.h index 4da22a4..6ae93b7 100644 --- a/shred.h +++ b/shred.h @@ -3,16 +3,25 @@ #include #include +#ifndef LARGEFILES /* - * 65,536 lines should be enough, but maybe someday this will be 32 bits. + * 65,536 lines should be enough, but make it possible to compile with 32 bits. * The point of making it a short is that the data sets can get quite large, * so we want the hash_t and sorthash_t structures to be small. */ typedef u_int16_t linenum_t; #define TONET htons #define FROMNET ntohs +#else +typedef u_int32_t linenum_t; +#define TONET htonl +#define FROMNET ntohl +#endif #define UNIQUE_FLAG (linenum_t)-1 /* 2s-complement assumption */ +/* use this to hold total line count of the entire source tree set */ +typedef u_int32_t linecount_t; + #define HASHSIZE 16 struct hash_t { -- 2.43.0