From c73491031726aec4d346b56cbf676d7c994c9a34 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 21 Nov 2017 15:45:21 +0000 Subject: [PATCH 1/7] Remove feature to compress GCproto.lineinfo The source code line number of each bytecode is now always stored in memory as a 32-bit number. Previously a simple compression scheme was used: an 8-bit number was used for <256 line functions, a 16-bit number for <64K line functions, and a 32-bit number otherwise. This specific optimization is not appropraite for RaptorJIT. On the one hand there will never be a practical benefit on any server platform because the amount of memory involved is miniscule, and on the other hand it complicates the JIT data structures that need to be understood by new maintainers and also supported in external tools like Studio. (Studio needs to understand the RaptorJIT data structures and in this case I decided to do that by removing code from RaptorJIT rather than adding code to Studio.) --- src/lj_bcread.c | 16 +++++----------- src/lj_debug.c | 15 +++------------ src/lj_obj.h | 2 +- src/lj_parse.c | 32 ++++++++------------------------ 4 files changed, 17 insertions(+), 48 deletions(-) diff --git a/src/lj_bcread.c b/src/lj_bcread.c index bd79849cd5..9fbfefcc3c 100644 --- a/src/lj_bcread.c +++ b/src/lj_bcread.c @@ -151,18 +151,12 @@ static uint32_t bcread_uleb128_33(LexState *ls) /* Read debug info of a prototype. */ static void bcread_dbg(LexState *ls, GCproto *pt, MSize sizedbg) { - void *lineinfo = (void *)proto_lineinfo(pt); + uint32_t *lineinfo = proto_lineinfo(pt); bcread_block(ls, lineinfo, sizedbg); /* Swap lineinfo if the endianess differs. */ - if (bcread_swap(ls) && pt->numline >= 256) { - MSize i, n = pt->sizebc-1; - if (pt->numline < 65536) { - uint16_t *p = (uint16_t *)lineinfo; - for (i = 0; i < n; i++) p[i] = (uint16_t)((p[i] >> 8)|(p[i] << 8)); - } else { - uint32_t *p = (uint32_t *)lineinfo; - for (i = 0; i < n; i++) p[i] = lj_bswap(p[i]); - } + if (bcread_swap(ls)) { + int i; + for (i = 0; i < pt->sizebc-1; i++) lineinfo[i] = lj_bswap(lineinfo[i]); } } @@ -367,7 +361,7 @@ GCproto *lj_bcread_proto(LexState *ls) pt->firstline = firstline; pt->numline = numline; if (sizedbg) { - MSize sizeli = (sizebc-1) << (numline < 256 ? 0 : numline < 65536 ? 1 : 2); + MSize sizeli = (sizebc-1) / sizeof(BCLine); setmref(pt->lineinfo, (char *)pt + ofsdbg); setmref(pt->uvinfo, (char *)pt + ofsdbg + sizeli); bcread_dbg(ls, pt, sizedbg); diff --git a/src/lj_debug.c b/src/lj_debug.c index 6b2b5e287d..ef08b7b2a3 100644 --- a/src/lj_debug.c +++ b/src/lj_debug.c @@ -113,12 +113,7 @@ BCLine lj_debug_line(GCproto *pt, BCPos pc) BCLine first = pt->firstline; if (pc == pt->sizebc) return first + pt->numline; if (pc-- == 0) return first; - if (pt->numline < 256) - return first + (BCLine)((const uint8_t *)lineinfo)[pc]; - else if (pt->numline < 65536) - return first + (BCLine)((const uint16_t *)lineinfo)[pc]; - else - return first + (BCLine)((const uint32_t *)lineinfo)[pc]; + return first + ((BCLine *)lineinfo)[pc]; } return 0; } @@ -497,16 +492,12 @@ int lj_debug_getinfo(lua_State *L, const char *what, lj_Debug *ar, int ext) if (isluafunc(fn)) { GCtab *t = lj_tab_new(L, 0, 0); GCproto *pt = funcproto(fn); - const void *lineinfo = proto_lineinfo(pt); + const uint32_t *lineinfo = proto_lineinfo(pt); if (lineinfo) { BCLine first = pt->firstline; - int sz = pt->numline < 256 ? 1 : pt->numline < 65536 ? 2 : 4; MSize i, szl = pt->sizebc-1; for (i = 0; i < szl; i++) { - BCLine line = first + - (sz == 1 ? (BCLine)((const uint8_t *)lineinfo)[i] : - sz == 2 ? (BCLine)((const uint16_t *)lineinfo)[i] : - (BCLine)((const uint32_t *)lineinfo)[i]); + BCLine line = first + lineinfo[i]; setboolV(lj_tab_setint(L, t, line), 1); } } diff --git a/src/lj_obj.h b/src/lj_obj.h index 4025177843..6991e02d1d 100644 --- a/src/lj_obj.h +++ b/src/lj_obj.h @@ -345,7 +345,7 @@ typedef struct GCproto { #define proto_chunkname(pt) (strref((pt)->chunkname)) #define proto_chunknamestr(pt) (strdata(proto_chunkname((pt)))) -#define proto_lineinfo(pt) (mref((pt)->lineinfo, const void)) +#define proto_lineinfo(pt) (mref((pt)->lineinfo, const uint32_t)) #define proto_uvinfo(pt) (mref((pt)->uvinfo, const uint8_t)) #define proto_varinfo(pt) (mref((pt)->varinfo, const uint8_t)) diff --git a/src/lj_parse.c b/src/lj_parse.c index 02a799b39c..353dc9383d 100644 --- a/src/lj_parse.c +++ b/src/lj_parse.c @@ -1352,7 +1352,7 @@ static void fs_fixup_uv1(FuncState *fs, GCproto *pt, uint16_t *uv) /* Prepare lineinfo for prototype. */ static size_t fs_prep_line(FuncState *fs, BCLine numline) { - return (fs->pc-1) << (numline < 256 ? 0 : numline < 65536 ? 1 : 2); + return (fs->pc-1) * sizeof(BCLine); } /* Fixup lineinfo for prototype. */ @@ -1365,28 +1365,12 @@ static void fs_fixup_line(FuncState *fs, GCproto *pt, pt->firstline = fs->linedefined; pt->numline = numline; setmref(pt->lineinfo, lineinfo); - if (LJ_LIKELY(numline < 256)) { - uint8_t *li = (uint8_t *)lineinfo; - do { - BCLine delta = base[i].line - first; - lua_assert(delta >= 0 && delta < 256); - li[i] = (uint8_t)delta; - } while (++i < n); - } else if (LJ_LIKELY(numline < 65536)) { - uint16_t *li = (uint16_t *)lineinfo; - do { - BCLine delta = base[i].line - first; - lua_assert(delta >= 0 && delta < 65536); - li[i] = (uint16_t)delta; - } while (++i < n); - } else { - uint32_t *li = (uint32_t *)lineinfo; - do { - BCLine delta = base[i].line - first; - lua_assert(delta >= 0); - li[i] = (uint32_t)delta; - } while (++i < n); - } + uint32_t *li = (uint32_t *)lineinfo; + do { + BCLine delta = base[i].line - first; + lua_assert(delta >= 0); + li[i] = (uint32_t)delta; + } while (++i < n); } /* Prepare variable info for prototype. */ @@ -1443,7 +1427,7 @@ static void fs_fixup_var(LexState *ls, GCproto *pt, uint8_t *p, size_t ofsvar) /* Initialize with empty debug info, if disabled. */ #define fs_prep_line(fs, numline) (UNUSED(numline), 0) #define fs_fixup_line(fs, pt, li, numline) \ - pt->firstline = pt->numline = 0, setmref((pt)->lineinfo, NULL) +pt->firstline = pt->numline = 0, setmref((pt)->lineinfo, NULL) #define fs_prep_var(ls, fs, ofsvar) (UNUSED(ofsvar), 0) #define fs_fixup_var(ls, pt, p, ofsvar) \ setmref((pt)->uvinfo, NULL), setmref((pt)->varinfo, NULL) From 5e673ead6c54b68876a58e43e11bc52696275e7e Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 21 Nov 2017 15:54:21 +0000 Subject: [PATCH 2/7] lj_obj.h: Fix comment about lineinfo (non)compression --- src/lj_obj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lj_obj.h b/src/lj_obj.h index 6991e02d1d..7cf74a77de 100644 --- a/src/lj_obj.h +++ b/src/lj_obj.h @@ -312,7 +312,7 @@ typedef struct GCproto { GCRef chunkname; /* Name of the chunk this function was defined in. */ BCLine firstline; /* First line of the function definition. */ BCLine numline; /* Number of lines for the function definition. */ - MRef lineinfo; /* Compressed map from bytecode ins. to source line. */ + MRef lineinfo; /* Map from bytecode ins. to source line. */ MRef uvinfo; /* Upvalue names. */ MRef varinfo; /* Names and compressed extents of local variables. */ } GCproto; From 7f4787acbcfa517788f873113ad39a38cf6dd3b1 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 21 Nov 2017 15:58:14 +0000 Subject: [PATCH 3/7] Fix bug in lj_bcread_proto() --- src/lj_bcread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lj_bcread.c b/src/lj_bcread.c index 9fbfefcc3c..b1e7498aab 100644 --- a/src/lj_bcread.c +++ b/src/lj_bcread.c @@ -361,7 +361,7 @@ GCproto *lj_bcread_proto(LexState *ls) pt->firstline = firstline; pt->numline = numline; if (sizedbg) { - MSize sizeli = (sizebc-1) / sizeof(BCLine); + MSize sizeli = (sizebc-1) * sizeof(BCLine); setmref(pt->lineinfo, (char *)pt + ofsdbg); setmref(pt->uvinfo, (char *)pt + ofsdbg + sizeli); bcread_dbg(ls, pt, sizedbg); From 17b3c9cdf26bb630acad286d61498171e8a96015 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 21 Nov 2017 15:59:57 +0000 Subject: [PATCH 4/7] lj_bcread.c: Fix warning --- src/lj_bcread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lj_bcread.c b/src/lj_bcread.c index b1e7498aab..3afd8250be 100644 --- a/src/lj_bcread.c +++ b/src/lj_bcread.c @@ -151,7 +151,7 @@ static uint32_t bcread_uleb128_33(LexState *ls) /* Read debug info of a prototype. */ static void bcread_dbg(LexState *ls, GCproto *pt, MSize sizedbg) { - uint32_t *lineinfo = proto_lineinfo(pt); + uint32_t *lineinfo = (uint32_t*)proto_lineinfo(pt); bcread_block(ls, lineinfo, sizedbg); /* Swap lineinfo if the endianess differs. */ if (bcread_swap(ls)) { From 7a4e25b94cb04d7c5178eb43f2bfd9c0380f20c5 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 28 Nov 2017 15:22:31 +0000 Subject: [PATCH 5/7] Add log of recorded bytecodes (BCLogRec) to jit_State --- src/lj_jit.h | 11 +++++++++++ src/lj_record.c | 9 +++++++++ src/lj_state.c | 3 +++ 3 files changed, 23 insertions(+) diff --git a/src/lj_jit.h b/src/lj_jit.h index 75f5804b01..a6c7292c02 100644 --- a/src/lj_jit.h +++ b/src/lj_jit.h @@ -295,6 +295,13 @@ typedef struct FoldState { IRIns right[2]; /* Instruction referenced by right operand. */ } FoldState; +/* Log entry for a bytecode that was recorded. */ +typedef struct BCRecLog { + GCproto *pt; + BCPos pos; + int32_t framedepth; +} BCRecLog; + /* JIT compiler state. */ typedef struct jit_State { GCtrace cur; /* Current trace. */ @@ -340,6 +347,10 @@ typedef struct jit_State { SnapEntry *snapmapbuf; /* Temp. snapshot map buffer. */ MSize sizesnapmap; /* Size of temp. snapshot map buffer. */ + BCRecLog *bclog; /* Start of of recorded bytecode log. */ + BCRecLog *bclognext; /* Next entry in the bytecode log. */ + BCRecLog *bcloglast; /* Last entry in the bytecode log. */ + PostProc postproc; /* Required post-processing after execution. */ uint8_t retryrec; /* Retry recording. */ diff --git a/src/lj_record.c b/src/lj_record.c index cce8cf34e0..8fc0e01ced 100644 --- a/src/lj_record.c +++ b/src/lj_record.c @@ -1864,6 +1864,13 @@ void lj_record_ins(jit_State *J) BCOp op; TRef ra, rb, rc; + if (J->bclognext < J->bcloglast) { + BCRecLog *log = J->bclognext++; + log->pt = J->pt; + log->pos = J->pt ? proto_bcpos(J->pt, J->pc) : -1; + log->framedepth = J->framedepth; + } + /* Perform post-processing action before recording the next instruction. */ if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) { switch (J->postproc) { @@ -2436,6 +2443,8 @@ void lj_record_setup(jit_State *J) J->bc_min = NULL; /* Means no limit. */ J->bc_extent = ~(MSize)0; + J->bclognext = J->bclog; + /* Emit instructions for fixed references. Also triggers initial IR alloc. */ emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno); for (i = 0; i <= 2; i++) { diff --git a/src/lj_state.c b/src/lj_state.c index f91f2c41dd..32f7ff1dc4 100644 --- a/src/lj_state.c +++ b/src/lj_state.c @@ -168,6 +168,7 @@ static void close_state(lua_State *L) lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef); lj_buf_free(g, &g->tmpbuf); lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue); + lj_mem_free(g, J->bclog, sizeof(BCRecLog)*65536); lj_mem_free(g, J->snapmapbuf, J->sizesnapmap); lj_mem_free(g, J->snapbuf, J->sizesnap); lj_mem_free(g, J->irbuf-REF_BIAS, 65536*sizeof(IRIns)); @@ -216,6 +217,8 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud) J->sizesnapmap = sizeof(SnapEntry)*65536; J->snapbuf = (SnapShot *)lj_mem_new(L, J->sizesnap); J->snapmapbuf = (SnapEntry *)lj_mem_new(L, J->sizesnapmap); + J->bclognext = J->bclog = (BCRecLog *)lj_mem_new(L, sizeof(BCRecLog)*65536); + J->bcloglast = J->bclog + 65535; IRIns *irbufmem = (IRIns *)lj_mem_new(L, sizeof(IRIns)*65536); if (irbufmem == NULL || J->snapbuf == NULL || J->snapmapbuf == NULL) return NULL; From 9b2b46751e7962dba31f11252927cb4b8b4b2239 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Wed, 29 Nov 2017 10:49:26 +0000 Subject: [PATCH 6/7] Minor update to jit_State bytecode log --- src/lj_jit.h | 10 +++++----- src/lj_record.c | 6 +++--- src/lj_state.c | 5 +++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/lj_jit.h b/src/lj_jit.h index a6c7292c02..66bb48d513 100644 --- a/src/lj_jit.h +++ b/src/lj_jit.h @@ -297,9 +297,9 @@ typedef struct FoldState { /* Log entry for a bytecode that was recorded. */ typedef struct BCRecLog { - GCproto *pt; - BCPos pos; - int32_t framedepth; + GCproto *pt; /* Prototype of bytecode function (or NULL). */ + BCPos pos; /* Position of bytecode in prototype. */ + int32_t framedepth; /* Frame depth when recorded. */ } BCRecLog; /* JIT compiler state. */ @@ -348,8 +348,8 @@ typedef struct jit_State { MSize sizesnapmap; /* Size of temp. snapshot map buffer. */ BCRecLog *bclog; /* Start of of recorded bytecode log. */ - BCRecLog *bclognext; /* Next entry in the bytecode log. */ - BCRecLog *bcloglast; /* Last entry in the bytecode log. */ + uint32_t nbclog; /* Number of logged bytecodes. */ + uint32_t maxbclog; /* Max entries in the bytecode log. */ PostProc postproc; /* Required post-processing after execution. */ uint8_t retryrec; /* Retry recording. */ diff --git a/src/lj_record.c b/src/lj_record.c index 8fc0e01ced..f4b3c0e0d5 100644 --- a/src/lj_record.c +++ b/src/lj_record.c @@ -1864,8 +1864,8 @@ void lj_record_ins(jit_State *J) BCOp op; TRef ra, rb, rc; - if (J->bclognext < J->bcloglast) { - BCRecLog *log = J->bclognext++; + if (J->nbclog < J->maxbclog) { + BCRecLog *log = &J->bclog[J->nbclog++]; log->pt = J->pt; log->pos = J->pt ? proto_bcpos(J->pt, J->pc) : -1; log->framedepth = J->framedepth; @@ -2443,7 +2443,7 @@ void lj_record_setup(jit_State *J) J->bc_min = NULL; /* Means no limit. */ J->bc_extent = ~(MSize)0; - J->bclognext = J->bclog; + J->nbclog = 0; /* Emit instructions for fixed references. Also triggers initial IR alloc. */ emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno); diff --git a/src/lj_state.c b/src/lj_state.c index 32f7ff1dc4..d5fccf6d4b 100644 --- a/src/lj_state.c +++ b/src/lj_state.c @@ -217,8 +217,9 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud) J->sizesnapmap = sizeof(SnapEntry)*65536; J->snapbuf = (SnapShot *)lj_mem_new(L, J->sizesnap); J->snapmapbuf = (SnapEntry *)lj_mem_new(L, J->sizesnapmap); - J->bclognext = J->bclog = (BCRecLog *)lj_mem_new(L, sizeof(BCRecLog)*65536); - J->bcloglast = J->bclog + 65535; + J->maxbclog = 65536; + J->bclog = (BCRecLog *)lj_mem_new(L, sizeof(BCRecLog)*J->maxbclog); + J->nbclog = 0; IRIns *irbufmem = (IRIns *)lj_mem_new(L, sizeof(IRIns)*65536); if (irbufmem == NULL || J->snapbuf == NULL || J->snapmapbuf == NULL) return NULL; From 796597d4c0f3666f6d4293258c6cd5b3f16fe3e0 Mon Sep 17 00:00:00 2001 From: Alexander Gall Date: Thu, 23 Nov 2017 10:53:49 +0100 Subject: [PATCH 7/7] lj_trace.c: unpatch blacklisted bytecodes when flushing traces The blacklist is cleared by going through all prototype objects on the heap marked to contain patched bytecodes and revert all instances of IFUNCF, IFORL, IITERL and ILOOP. (cherry picked from commit 8beb311332a6c95ab1d2934f292f41563b878edb) --- src/lj_trace.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lj_trace.c b/src/lj_trace.c index ade66b27d1..c739ebe106 100644 --- a/src/lj_trace.c +++ b/src/lj_trace.c @@ -251,6 +251,15 @@ int lj_trace_flushall(lua_State *L) } J->cur.traceno = 0; J->freetrace = 0; + /* Unpatch blacklisted byte codes. */ + GCRef *p = &(G(L)->gc.root); + GCobj *o; + while ((o = gcref(*p)) != NULL) { + if (o->gch.gct == ~LJ_TPROTO) { + lj_trace_reenableproto(gco2pt(o)); + } + p = &o->gch.nextgc; + } /* Clear penalty cache. */ memset(J->penalty, 0, sizeof(J->penalty)); /* Free the whole machine code and invalidate all exit stub groups. */