From 5a948c47158e63137a06e8588571084aa4166cda Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 10 May 2008 22:03:48 -0400 Subject: [PATCH] Split output of the OIL to C compiler --- Makefile.in | 30 ++++------- NEWS.txt | 20 ++++--- src/oil.h | 30 ++++++++++- src/oil.y | 151 +++++++++++++++++++++++++++++++++------------------- 4 files changed, 148 insertions(+), 83 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8de57b6..ba020fe 100644 --- a/Makefile.in +++ b/Makefile.in @@ -107,8 +107,8 @@ include/ebcdic.bin:src/ebcdic.bin include/latin1.bin:src/latin1.bin -cp src/latin1.bin include/latin1.bin -bin/ick@EXEEXT@: temp/parser.@OBJEXT@ temp/lexer.@OBJEXT@ temp/feh2.@OBJEXT@ temp/dekludge.@OBJEXT@ temp/idiotism.@OBJEXT@ temp/ick_lose.@OBJEXT@ temp/fiddle.@OBJEXT@ temp/perpet.@OBJEXT@ temp/uncommon.@OBJEXT@ - $(CC) temp/perpet.@OBJEXT@ temp/parser.@OBJEXT@ temp/lexer.@OBJEXT@ temp/feh2.@OBJEXT@ temp/dekludge.@OBJEXT@ temp/idiotism.@OBJEXT@ temp/ick_lose.@OBJEXT@ temp/fiddle.@OBJEXT@ temp/uncommon.@OBJEXT@ $(LDFLAGS) -o bin/ick@EXEEXT@ +bin/ick@EXEEXT@: temp/parser.@OBJEXT@ temp/lexer.@OBJEXT@ temp/feh2.@OBJEXT@ temp/dekludge.@OBJEXT@ temp/oilout-m.@OBJEXT@ temp/ick_lose.@OBJEXT@ temp/fiddle.@OBJEXT@ temp/perpet.@OBJEXT@ temp/uncommon.@OBJEXT@ + $(CC) temp/perpet.@OBJEXT@ temp/parser.@OBJEXT@ temp/lexer.@OBJEXT@ temp/feh2.@OBJEXT@ temp/dekludge.@OBJEXT@ temp/oilout*.@OBJEXT@ temp/ick_lose.@OBJEXT@ temp/fiddle.@OBJEXT@ temp/uncommon.@OBJEXT@ $(LDFLAGS) -o bin/ick@EXEEXT@ bin/convickt@EXEEXT@: temp/convickt.@OBJEXT@ temp/clc-cset.@OBJEXT@ temp/uncommon.@OBJEXT@ $(CC) temp/convickt.@OBJEXT@ temp/clc-cset.@OBJEXT@ temp/uncommon.@OBJEXT@ $(LDFLAGS) -o bin/convickt@EXEEXT@ @@ -143,24 +143,12 @@ temp/lexer.@OBJEXT@: temp/lexer.c temp/parser.h src/ick.h src/ick_lose.h temp/co $(CC) $(CFLAGS) -c -o temp/lexer.@OBJEXT@ temp/lexer.c temp/feh2.@OBJEXT@: src/feh2.c src/feh.h temp/parser.h src/ick.h src/fiddle.h src/ick_lose.h src/sizes.h temp/config.h $(CC) $(CFLAGS) -c -o temp/feh2.@OBJEXT@ src/feh2.c -temp/idiotism.c: src/idiotism.oil temp/oil@EXEEXT@ - temp/oil@EXEEXT@ < src/idiotism.oil > temp/idiotism.c -# There is a situation that some operating systems can get into known as memory -# overcommit. Linux (talking specifically about the kernel) can reach this -# situation in its default configuration, where it has been reporting to malloc -# that memory is available when in fact it isn't, and reaches the situation when -# the computer is actually completely out of memory but programs on it believe -# that they aren't. For some reason, using the default options, gcc can somehow -# cause memory overcommit when acting on a file as big as idiotism.c. The -# solution here, therefore, is to put two protections to prevent this happening; -# first, turn off optimisation specifically for that file, and second, put an -# artificial memory limit on the compilation so that no more than 128 megabytes -# can be allocated, using the ulimit feature on bash. As memory overcommit can -# make a system unusable, and as it can be preceded by 'thrashing' (hugely -# rapid swaps to disk in an attempt to get more memory) that can also make a -# system unusable, steps to avoid this are necessary. -temp/idiotism.@OBJEXT@: temp/idiotism.c src/feh.h temp/parser.h src/ick.h src/fiddle.h src/ick_lose.h src/sizes.h temp/config.h - bash -c 'ulimit -Sv 131072; ulimit -Sd 131072; $(CC) $(CFLAGS) -c -o temp/idiotism.@OBJEXT@ -O0 temp/idiotism.c' +temp/oilout-m.c: src/idiotism.oil temp/oil@EXEEXT@ + bash -c 'cd temp && rm -f oilout*.c && ./oil@EXEEXT@ < ../src/idiotism.oil && cd ..' +temp/oilout-m.@OBJEXT@: temp/oilout-m.c src/feh.h temp/parser.h src/ick.h src/fiddle.h src/ick_lose.h src/sizes.h temp/config.h + -rm -f temp/oilout*.@OBJEXT@ + $(CC) $(CFLAGS) -c temp/oilout*.c + -mv oilout*.@OBJEXT@ temp temp/dekludge.@OBJEXT@: src/dekludge.c src/feh.h temp/parser.h src/ick.h src/fiddle.h src/ick_lose.h src/sizes.h temp/config.h $(CC) $(CFLAGS) -c -o temp/dekludge.@OBJEXT@ src/dekludge.c temp/perpet.@OBJEXT@: src/perpet.c temp/parser.h src/ick.h src/ick_lose.h src/sizes.h src/uncommon.h temp/config.h @@ -303,7 +291,7 @@ clean: distr: -rm -f temp/*.@OBJEXT@ temp/oil@EXEEXT@ - -rm -f temp/idiotism.c temp/bin2c@EXEEXT@ temp/cset-?.c + -rm -f temp/oilout*.c temp/bin2c@EXEEXT@ temp/cset-?.c -rm -f bin/* lib/* include/* distclean: distr diff --git a/NEWS.txt b/NEWS.txt index dac18a2..6f55f81 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -53,23 +53,31 @@ Changes by Joris Huizer 10. The -g option now causes optimisation of the output compiled C to be turned off. +Bugs reported by the Debian autobuilders and Joey Hess + +11. The build of the optimiser idioms is now split over many files, to reduce + the memory usage of gcc when it tries to compile the giant output C file. + This was a problem on ia64. (The ulimit method of preventing thrashing has + been removed, as this method works better; likewise, the optimiser + optimisation now works again.) + Changes by Alex Smith -11. convickt can now produce its output as INTERCAL initialisation of a tail +12. convickt can now produce its output as INTERCAL initialisation of a tail array. (This doesn't auto-offset between the different characters; it is therefore mostly useful for producing Baudot arrays for CLC-INTERCAL-style IO.) -12. The function clock_gettime is now supported by the yuk profiler as an +13. The function clock_gettime is now supported by the yuk profiler as an alternative method of getting a high-resolution clock for profiling (it is available on at least Linux). -13. Upgraded the autoconfiguration to a substantially newer version of +14. Upgraded the autoconfiguration to a substantially newer version of autoconf. Hopefully the same script works on all operating systems now, because I've removed the one that configures for DOS (which was previously separate). -14. Added support for computed CREATE (the computation is done at create-time, +15. Added support for computed CREATE (the computation is done at create-time, rather than when the unknown statement is encountered); partly because this was rather easy, but mostly because it can be used to simulate a computed NEXT, thus making the example expansion library compunex (which I @@ -81,10 +89,10 @@ Changes by Alex Smith features of NEXT, anyway, rather than the small subset COMPUNEX could manage.) -15. Plugged a couple of small memory leaks; however, there are still several +16. Plugged a couple of small memory leaks; however, there are still several much bigger memory leaks that could do with being fixed. -16. About the version number: this release is mostly bugfixing, but does add a +17. About the version number: this release is mostly bugfixing, but does add a few new features, so I wasn't sure whether to increment the major or the minor version number. In the end, I used the old INTERCAL maxim 'when in doubt, do both'. diff --git a/src/oil.h b/src/oil.h index 2728e29..6aa855f 100644 --- a/src/oil.h +++ b/src/oil.h @@ -1,7 +1,35 @@ -/* Name mangling for compiled OIL programs. +/* Header file for compiled OIL programs + Does name mangling, defines macros Copyright (C) Alex Smith 2008 See oil.y for copyright conditions. */ +#include +#include +#include +#include +#include "sizes.h" +#include "ick.h" +#include "parser.h" +#include "fiddle.h" +#include "ick_lose.h" +#include "feh.h" + +#define OPTING(x) \ + if(optdebug == 2) \ + { \ + explexpr(optdebugnode,stderr); \ + putc('\n',stderr); \ + } \ + if(optdebug == 3) \ + { \ + prexpr(optdebugnode,stderr,0); \ + putc('\n',stderr); \ + } \ + if(optdebug) fprintf(stderr,"[%s]",#x); \ + if(optdebug >= 2) putc('\n',stderr); \ + opted = 1; +#define MAYBENEWNODE(n) if(!(n)) (n)=newnode(); + #define Base ick_Base #define Large_digits ick_Large_digits #define Max_large ick_Max_large diff --git a/src/oil.y b/src/oil.y index d966e71..6c02d6e 100644 --- a/src/oil.y +++ b/src/oil.y @@ -1,7 +1,7 @@ /***************************************************************************** -NAME - oil.y -- compiler for Optimiser Idiom Language files +NAME + oil.y -- compiler for Optimizer Idiom Language files LICENSE TERMS Copyright (C) 2007 Alex Smith @@ -56,7 +56,7 @@ char* strdup(char* s) /* #define YYDEBUG 1 -int yydebug=1; +int yydebug=1; */ /* Each semantic value represents either a constraint on a node that needs to @@ -94,23 +94,26 @@ struct ickstype #define YYSTYPE YYSTYPE #define MAXOPTNAMELEN 64 - char optname[MAXOPTNAMELEN]="undefined"; +char optname[MAXOPTNAMELEN]="undefined"; - typedef struct ickstype *YYSTYPE; +typedef struct ickstype *YYSTYPE; - void treedepthup(YYSTYPE, mybool); - void treefree(YYSTYPE); - void gennodepath(unsigned, unsigned long); - mybool treeshapecond(YYSTYPE, mybool); - YYSTYPE treenscheck(YYSTYPE, YYSTYPE, int); - void treecxcond(YYSTYPE); - void treerepcount(YYSTYPE, int*); - void treerepgen(YYSTYPE, YYSTYPE*, int*); +void splitend(); +void splitstart(); - int countgetchar(void); - int countungetc(int, FILE*); - int cgccol; - int cgcrow; +void treedepthup(YYSTYPE, mybool); +void treefree(YYSTYPE); +void gennodepath(unsigned, unsigned long); +mybool treeshapecond(YYSTYPE, mybool); +YYSTYPE treenscheck(YYSTYPE, YYSTYPE, int); +void treecxcond(YYSTYPE); +void treerepcount(YYSTYPE, int*); +void treerepgen(YYSTYPE, YYSTYPE*, int*); + +int countgetchar(void); +int countungetc(int, FILE*); +int cgccol; +int cgcrow; /* #defines for chaining together template expressions; here, s is the type of expression (e.g. select, bitwise and, unary and) that's chaining the @@ -151,8 +154,14 @@ struct ickstype } while(0) /* Error handling and lexing */ - int yylex(void); - int yyerror(char const *); +int yylex(void); +int yyerror(char const *); + +/* Split the output file */ +#define SPLITMAX 20 +int splitcount=SPLITMAX; +int filenumber=0; +mybool inloop=0; %} /* Various conditions can come out from the lexer. The most common is a char, @@ -181,7 +190,7 @@ struct ickstype The other things that can come out from the lexer are sparks, ears, and parentheses (any mix, we're not fussy). */ - + %token LEXERLEAF %% @@ -192,6 +201,15 @@ optimization: template '-' '>' replacement { static YYSTYPE tempmem[10]; static int replcount[10]; + /* Handle splitting the file. */ + if(splitcount) splitcount--; + if(!splitcount && !inloop) + { + splitcount=SPLITMAX; + splitend(); + ++filenumber; + splitstart(); + } /* This is where we actually generate the optimizer code. */ /* Tree-shape and is-constant conditions */ printf(" checknodeactbits(np);\n"); @@ -232,7 +250,7 @@ optimization: template '-' '>' replacement treerepgen($4,tempmem,replcount); printf(" nodefree(np->lval); nodefree(np->rval);\n"); printf(" tempw=np->width; *np=*tp; np->width=tempw; free(tp);\n"); - printf(" } while(0);\n\n"); + printf(" } while(0);\n\n"); /* Free the template and replacement now they're finished being used. */ treefree($1); treefree($4); @@ -247,8 +265,9 @@ optimization: template '-' '>' replacement } printf(" r=%luLU; while(r<=%luLU) {\n",$2->c,$4->c); free($2); free($4); + inloop=1; } -| '>' {printf(" r++;\n }\n");}; +| '>' {printf(" r++;\n }\n"); inloop=0;}; template: expr3 ; @@ -689,41 +708,30 @@ int countungetc(int c, FILE* f) return c; } -int main(void) +void splitstart() { - int e; - /* Technically speaking, this string exceeds the longest permitted string - that can be written as a constant string. Compilers are unlikely to care, - though; if yours does, just break this into separate printfs. */ - printf("%s", - "/* machine-generated code; modify the corresponding .oil file\n" - " and recompile, not this one, to change this file */\n\n" - "#include \n#include \n#include \n" - "#include \n#include \"oil.h\"\n" - "#include \"sizes.h\"\n#include \"ick.h\"\n#include \"parser.h\"\n" - "#include \"fiddle.h\"\n#include \"ick_lose.h\"\n#include \"feh.h\"\n\n" - "#define OPTING(x) if(optdebug == 2) {\\\n" - " explexpr(optdebugnode,stderr);\\\n" - " putc('\\n',stderr); } \\\n" - " if(optdebug == 3) {\\\n" - " prexpr(optdebugnode,stderr,0);\\\n" - " putc('\\n',stderr); } \\\n" - " if(optdebug) fprintf(stderr,\"[%s]\",#x);\\\n" - " if(optdebug >= 2) putc('\\n',stderr);\\\n" - " opted = 1;\n\n" - "#define MAYBENEWNODE(n) if(!(n)) (n)=newnode();\n\n" - "int optimize_pass1(node *np)\n{\n int opted=0;\n" - " unsigned long c,c1,c2,c3,c4,c5,c6,c7,c8,c9;\n" - " unsigned long x,x1,x2,x3,x4,x5,x6,x7,x8,x9,r;\n" + static char fname[]="oilout00.c"; + if(filenumber>255) + { + filenumber=255; + fprintf(stdout,"Input file too long.\n"); + } + sprintf(fname,"oilout%02x.c",filenumber); + freopen(fname,"w",stdout); + puts("/* Automatically generated output, edit source and recompile to " + "change */"); + printf("#include \"oil.h\"\n" + "int optimize_pass1_%x(node *np)\n" + "{" + " int opted=0;\n" + " unsigned long c,c1,c2,c3,c4,c5,c6,c7,c8,c9;\n" + " unsigned long x,x1,x2,x3,x4,x5,x6,x7,x8,x9,r;\n" " int tempw;\n" - " node *tp;\n" - " if(!np) return 0;\n" - " if(np->lval) opted|=optimize_pass1(np->lval);\n" - " if(np->rval) opted|=optimize_pass1(np->rval);\n"); - cgccol=0; - cgcrow=1; - e=yyparse(); - while(tfi--) free(tofree[tfi]); + " node *tp;\n", filenumber); +} + +void splitend() +{ /* Disabling warnings about unused variables. gcc will optimize this right out, and in any case the raise(SIGSEGV) will be unreachable (but will cause a pretty recognizable error because it'll be caught by the handler @@ -734,5 +742,38 @@ int main(void) " if(c+c1+c2+c3+c4+c5+c6+c7+c8+c9+r+\n" " x+x1+x2+x3+x4+x5+x6+x7+x8+x9) raise(SIGSEGV);\n"); printf(" return opted;\n}\n"); + fclose(stdout); +} + +int main(void) +{ + int e,i; + /* + " if(!np) return 0;\n" + " if(np->lval) opted|=optimize_pass1(np->lval);\n" + " if(np->rval) opted|=optimize_pass1(np->rval);\n" + */ + splitstart(); + cgccol=0; + cgcrow=1; + e=yyparse(); + while(tfi--) free(tofree[tfi]); + splitend(); + freopen("oilout-m.c","w",stdout); + puts("/* Automatically generated output, edit source and recompile to " + "change */"); + puts("#include \"ick.h\""); + i=filenumber+1; + while(i--) printf("extern int optimize_pass1_%x(node*);\n",i); + puts("int optimize_pass1(node* np)\n" + "{\n" + " int opted=0;\n" + " if(!np) return 0;\n" + " if(np->lval) opted|=optimize_pass1(np->lval);\n" + " if(np->rval) opted|=optimize_pass1(np->rval);"); + i=filenumber+1; + while(i--) printf(" opted|=optimize_pass1_%x(np);\n",i); + puts(" return opted;\n" + "}"); return e; } -- 2.43.0