perf: pre-compile embedded Lua source into bytecode (#16631)

The Lua modules that make up vim.lua are embedded as raw source files into the
nvim binary. These sources are loaded by the Lua runtime on startuptime. We can
pre-compile these sources into Lua bytecode before embedding them into the
binary, which minimizes the size of the binary and improves startuptime.
This commit is contained in:
Gregory Anders 2021-12-16 09:27:39 -07:00 committed by GitHub
parent 56fa08b458
commit 4240ce8eb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 136 additions and 79 deletions

View File

@ -543,6 +543,24 @@ endif()
message(STATUS "Using Lua interpreter: ${LUA_PRG}")
if(NOT WIN32)
if(PREFER_LUA)
foreach(CURRENT_LUAC_PRG luac5.1 luac)
find_program(_CHECK_LUAC_PRG ${CURRENT_LUAC_PRG})
if(_CHECK_LUAC_PRG)
set(LUAC_PRG "${_CHECK_LUAC_PRG} -s -o - %s" CACHE STRING "Format for compiling to Lua bytecode")
break()
endif()
endforeach()
elseif(LUA_PRG MATCHES "luajit")
set(LUAC_PRG "${LUA_PRG} -b -s %s -" CACHE STRING "Format for compiling to Lua bytecode")
endif()
endif()
if(LUAC_PRG)
message(STATUS "Using Lua compiler: ${LUAC_PRG}")
endif()
# Setup busted.
find_program(BUSTED_PRG NAMES busted busted.bat)
find_program(BUSTED_LUA_PRG busted-lua)

View File

@ -326,7 +326,9 @@ add_custom_command(
add_custom_command(
OUTPUT ${VIM_MODULE_FILE}
COMMAND ${LUA_PRG} ${CHAR_BLOB_GENERATOR} ${VIM_MODULE_FILE}
COMMAND ${CMAKE_COMMAND} -E env
"LUAC_PRG=${LUAC_PRG}"
${LUA_PRG} ${CHAR_BLOB_GENERATOR} -c ${VIM_MODULE_FILE}
${LUA_VIM_MODULE_SOURCE} vim_module
${LUA_SHARED_MODULE_SOURCE} shared_module
${LUA_INSPECT_MODULE_SOURCE} inspect_module
@ -339,6 +341,7 @@ add_custom_command(
${LUA_INSPECT_MODULE_SOURCE}
${LUA_F_MODULE_SOURCE}
${LUA_META_MODULE_SOURCE}
VERBATIM
)
list(APPEND NVIM_GENERATED_SOURCES

View File

@ -1,12 +1,26 @@
if arg[1] == '--help' then
print('Usage:')
print(' '..arg[0]..' target source varname [source varname]...')
print(' '..arg[0]..' [-c] target source varname [source varname]...')
print('')
print('Generates C file with big uint8_t blob.')
print('Blob will be stored in a static const array named varname.')
os.exit()
end
-- Recognized options:
-- -c compile Lua bytecode
local options = {}
while true do
local opt = string.match(arg[1], "^-(%w)")
if not opt then
break
end
options[opt] = true
table.remove(arg, 1)
end
assert(#arg >= 3 and (#arg - 1) % 2 == 0)
local target_file = arg[1] or error('Need a target file')
@ -23,11 +37,25 @@ for argi = 2, #arg, 2 do
end
varnames[varname] = source_file
local source = io.open(source_file, 'r')
or error(string.format("source_file %q doesn't exist", source_file))
target:write(('static const uint8_t %s[] = {\n'):format(varname))
local output
if options.c then
local luac = os.getenv("LUAC_PRG")
if luac then
output = io.popen(luac:format(source_file), "r"):read("*a")
else
print("LUAC_PRG is undefined")
end
end
if not output then
local source = io.open(source_file, "r")
or error(string.format("source_file %q doesn't exist", source_file))
output = source:read("*a")
source:close()
end
local num_bytes = 0
local MAX_NUM_BYTES = 15 -- 78 / 5: maximum number of bytes on one line
target:write(' ')
@ -41,19 +69,13 @@ for argi = 2, #arg, 2 do
end
end
for line in source:lines() do
for i = 1, string.len(line) do
local byte = line:byte(i)
assert(byte ~= 0)
target:write(string.format(' %3u,', byte))
increase_num_bytes()
end
target:write(string.format(' %3u,', string.byte('\n', 1)))
for i = 1, string.len(output) do
local byte = output:byte(i)
target:write(string.format(' %3u,', byte))
increase_num_bytes()
end
target:write(' 0};\n')
source:close()
target:write(' 0};\n')
end
target:close()

View File

@ -404,9 +404,9 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
{
const char *code = (char *)&shared_module[0];
if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/shared.lua")
if (luaL_loadbuffer(lstate, code, sizeof(shared_module) - 1, "@vim/shared.lua")
|| nlua_pcall(lstate, 0, 0)) {
nlua_error(lstate, _("E5106: Error while creating shared module: %.*s"));
nlua_error(lstate, _("E5106: Error while creating shared module: %.*s\n"));
return 1;
}
}
@ -416,18 +416,18 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
lua_getfield(lstate, -1, "loaded"); // [package, loaded]
const char *code = (char *)&inspect_module[0];
if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/inspect.lua")
if (luaL_loadbuffer(lstate, code, sizeof(inspect_module) - 1, "@vim/inspect.lua")
|| nlua_pcall(lstate, 0, 1)) {
nlua_error(lstate, _("E5106: Error while creating inspect module: %.*s"));
nlua_error(lstate, _("E5106: Error while creating inspect module: %.*s\n"));
return 1;
}
// [package, loaded, inspect]
lua_setfield(lstate, -2, "vim.inspect"); // [package, loaded]
code = (char *)&lua_F_module[0];
if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/F.lua")
if (luaL_loadbuffer(lstate, code, sizeof(lua_F_module) - 1, "@vim/F.lua")
|| nlua_pcall(lstate, 0, 1)) {
nlua_error(lstate, _("E5106: Error while creating vim.F module: %.*s"));
nlua_error(lstate, _("E5106: Error while creating vim.F module: %.*s\n"));
return 1;
}
// [package, loaded, module]
@ -438,9 +438,9 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
{
const char *code = (char *)&vim_module[0];
if (luaL_loadbuffer(lstate, code, strlen(code), "@vim.lua")
if (luaL_loadbuffer(lstate, code, sizeof(vim_module) - 1, "@vim.lua")
|| nlua_pcall(lstate, 0, 0)) {
nlua_error(lstate, _("E5106: Error while creating vim module: %.*s"));
nlua_error(lstate, _("E5106: Error while creating vim module: %.*s\n"));
return 1;
}
}
@ -450,9 +450,9 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
lua_getfield(lstate, -1, "loaded"); // [package, loaded]
const char *code = (char *)&lua_meta_module[0];
if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/_meta.lua")
if (luaL_loadbuffer(lstate, code, sizeof(lua_meta_module) - 1, "@vim/_meta.lua")
|| nlua_pcall(lstate, 0, 1)) {
nlua_error(lstate, _("E5106: Error while creating vim._meta module: %.*s"));
nlua_error(lstate, _("E5106: Error while creating vim._meta module: %.*s\n"));
return 1;
}
// [package, loaded, module]

View File

@ -131,9 +131,9 @@ describe('lua stdlib', function()
eq(false, funcs.luaeval('vim.startswith("123", "2")'))
eq(false, funcs.luaeval('vim.startswith("123", "1234")'))
eq("Error executing lua: vim/shared.lua:0: prefix: expected string, got nil",
matches("prefix: expected string, got nil",
pcall_err(exec_lua, 'return vim.startswith("123", nil)'))
eq("Error executing lua: vim/shared.lua:0: s: expected string, got nil",
matches("s: expected string, got nil",
pcall_err(exec_lua, 'return vim.startswith(nil, "123")'))
end)
@ -147,9 +147,9 @@ describe('lua stdlib', function()
eq(false, funcs.luaeval('vim.endswith("123", "2")'))
eq(false, funcs.luaeval('vim.endswith("123", "1234")'))
eq("Error executing lua: vim/shared.lua:0: suffix: expected string, got nil",
matches("suffix: expected string, got nil",
pcall_err(exec_lua, 'return vim.endswith("123", nil)'))
eq("Error executing lua: vim/shared.lua:0: s: expected string, got nil",
matches("s: expected string, got nil",
pcall_err(exec_lua, 'return vim.endswith(nil, "123")'))
end)
@ -220,9 +220,9 @@ describe('lua stdlib', function()
eq({"yy","xx"}, exec_lua("return test_table"))
-- Validates args.
eq('Error executing lua: vim.schedule: expected function',
matches('vim.schedule: expected function',
pcall_err(exec_lua, "vim.schedule('stringly')"))
eq('Error executing lua: vim.schedule: expected function',
matches('vim.schedule: expected function',
pcall_err(exec_lua, "vim.schedule()"))
exec_lua([[
@ -232,7 +232,7 @@ describe('lua stdlib', function()
]])
feed("<cr>")
eq('Error executing vim.schedule lua callback: [string "<nvim>"]:2: big failure\nvery async', remove_trace(eval("v:errmsg")))
matches('big failure\nvery async', remove_trace(eval("v:errmsg")))
local screen = Screen.new(60,5)
screen:set_default_attr_ids({
@ -300,16 +300,16 @@ describe('lua stdlib', function()
}
for _, t in ipairs(loops) do
eq("Error executing lua: vim/shared.lua:0: Infinite loop detected", pcall_err(split, t[1], t[2]))
matches("Infinite loop detected", pcall_err(split, t[1], t[2]))
end
-- Validates args.
eq(true, pcall(split, 'string', 'string'))
eq('Error executing lua: vim/shared.lua:0: s: expected string, got number',
matches('s: expected string, got number',
pcall_err(split, 1, 'string'))
eq('Error executing lua: vim/shared.lua:0: sep: expected string, got number',
matches('sep: expected string, got number',
pcall_err(split, 'string', 1))
eq('Error executing lua: vim/shared.lua:0: kwargs: expected table, got number',
matches('kwargs: expected table, got number',
pcall_err(split, 'string', 'string', 1))
end)
@ -330,7 +330,7 @@ describe('lua stdlib', function()
end
-- Validates args.
eq('Error executing lua: vim/shared.lua:0: s: expected string, got number',
matches('s: expected string, got number',
pcall_err(trim, 2))
end)
@ -410,7 +410,7 @@ describe('lua stdlib', function()
return getmetatable(t2) == mt
]]))
eq('Error executing lua: vim/shared.lua:0: Cannot deepcopy object of type thread',
matches('Cannot deepcopy object of type thread',
pcall_err(exec_lua, [[
local thread = coroutine.create(function () return 0 end)
local t = {thr = thread}
@ -423,7 +423,7 @@ describe('lua stdlib', function()
eq('foo%%%-bar', exec_lua([[return vim.pesc(vim.pesc('foo-bar'))]]))
-- Validates args.
eq('Error executing lua: vim/shared.lua:0: s: expected string, got number',
matches('s: expected string, got number',
pcall_err(exec_lua, [[return vim.pesc(2)]]))
end)
@ -548,19 +548,19 @@ describe('lua stdlib', function()
return c.x.a == 1 and c.x.b == 2 and c.x.c == nil and count == 1
]]))
eq('Error executing lua: vim/shared.lua:0: invalid "behavior": nil',
matches('invalid "behavior": nil',
pcall_err(exec_lua, [[
return vim.tbl_extend()
]])
)
eq('Error executing lua: vim/shared.lua:0: wrong number of arguments (given 1, expected at least 3)',
matches('wrong number of arguments %(given 1, expected at least 3%)',
pcall_err(exec_lua, [[
return vim.tbl_extend("keep")
]])
)
eq('Error executing lua: vim/shared.lua:0: wrong number of arguments (given 2, expected at least 3)',
matches('wrong number of arguments %(given 2, expected at least 3%)',
pcall_err(exec_lua, [[
return vim.tbl_extend("keep", {})
]])
@ -661,19 +661,19 @@ describe('lua stdlib', function()
return vim.tbl_deep_extend("force", a, b)
]]), {a = 123 })
eq('Error executing lua: vim/shared.lua:0: invalid "behavior": nil',
matches('invalid "behavior": nil',
pcall_err(exec_lua, [[
return vim.tbl_deep_extend()
]])
)
eq('Error executing lua: vim/shared.lua:0: wrong number of arguments (given 1, expected at least 3)',
matches('wrong number of arguments %(given 1, expected at least 3%)',
pcall_err(exec_lua, [[
return vim.tbl_deep_extend("keep")
]])
)
eq('Error executing lua: vim/shared.lua:0: wrong number of arguments (given 2, expected at least 3)',
matches('wrong number of arguments %(given 2, expected at least 3%)',
pcall_err(exec_lua, [[
return vim.tbl_deep_extend("keep", {})
]])
@ -706,7 +706,7 @@ describe('lua stdlib', function()
it('vim.list_extend', function()
eq({1,2,3}, exec_lua [[ return vim.list_extend({1}, {2,3}) ]])
eq('Error executing lua: vim/shared.lua:0: src: expected table, got nil',
matches('src: expected table, got nil',
pcall_err(exec_lua, [[ return vim.list_extend({1}, nil) ]]))
eq({1,2}, exec_lua [[ return vim.list_extend({1}, {2;a=1}) ]])
eq(true, exec_lua [[ local a = {1} return vim.list_extend(a, {2;a=1}) == a ]])
@ -730,7 +730,7 @@ describe('lua stdlib', function()
assert(vim.deep_equal(a, { A = 1; [1] = 'A'; }))
vim.tbl_add_reverse_lookup(a)
]]
matches('^Error executing lua: vim/shared%.lua:0: The reverse lookup found an existing value for "[1A]" while processing key "[1A]"$',
matches('The reverse lookup found an existing value for "[1A]" while processing key "[1A]"$',
pcall_err(exec_lua, code))
end)
@ -771,7 +771,7 @@ describe('lua stdlib', function()
end)
it('vim.fn should error when calling API function', function()
eq('Error executing lua: vim.lua:0: Tried to call API function with vim.fn: use vim.api.nvim_get_current_line instead',
matches('Tried to call API function with vim.fn: use vim.api.nvim_get_current_line instead',
pcall_err(exec_lua, "vim.fn.nvim_get_current_line()"))
end)
@ -907,37 +907,37 @@ describe('lua stdlib', function()
exec_lua("vim.validate{arg1={{}, 't' }, arg2={ 'foo', 's' }}")
exec_lua("vim.validate{arg1={2, function(a) return (a % 2) == 0 end, 'even number' }}")
eq('Error executing lua: [string "<nvim>"]:0: opt[1]: expected table, got number',
matches('expected table, got number',
pcall_err(exec_lua, "vim.validate{ 1, 'x' }"))
eq('Error executing lua: [string "<nvim>"]:0: invalid type name: x',
matches('invalid type name: x',
pcall_err(exec_lua, "vim.validate{ arg1={ 1, 'x' }}"))
eq('Error executing lua: [string "<nvim>"]:0: invalid type name: 1',
matches('invalid type name: 1',
pcall_err(exec_lua, "vim.validate{ arg1={ 1, 1 }}"))
eq('Error executing lua: [string "<nvim>"]:0: invalid type name: nil',
matches('invalid type name: nil',
pcall_err(exec_lua, "vim.validate{ arg1={ 1 }}"))
-- Validated parameters are required by default.
eq('Error executing lua: [string "<nvim>"]:0: arg1: expected string, got nil',
matches('arg1: expected string, got nil',
pcall_err(exec_lua, "vim.validate{ arg1={ nil, 's' }}"))
-- Explicitly required.
eq('Error executing lua: [string "<nvim>"]:0: arg1: expected string, got nil',
matches('arg1: expected string, got nil',
pcall_err(exec_lua, "vim.validate{ arg1={ nil, 's', false }}"))
eq('Error executing lua: [string "<nvim>"]:0: arg1: expected table, got number',
matches('arg1: expected table, got number',
pcall_err(exec_lua, "vim.validate{arg1={1, 't'}}"))
eq('Error executing lua: [string "<nvim>"]:0: arg2: expected string, got number',
matches('arg2: expected string, got number',
pcall_err(exec_lua, "vim.validate{arg1={{}, 't'}, arg2={1, 's'}}"))
eq('Error executing lua: [string "<nvim>"]:0: arg2: expected string, got nil',
matches('arg2: expected string, got nil',
pcall_err(exec_lua, "vim.validate{arg1={{}, 't'}, arg2={nil, 's'}}"))
eq('Error executing lua: [string "<nvim>"]:0: arg2: expected string, got nil',
matches('arg2: expected string, got nil',
pcall_err(exec_lua, "vim.validate{arg1={{}, 't'}, arg2={nil, 's'}}"))
eq('Error executing lua: [string "<nvim>"]:0: arg1: expected even number, got 3',
matches('arg1: expected even number, got 3',
pcall_err(exec_lua, "vim.validate{arg1={3, function(a) return a == 1 end, 'even number'}}"))
eq('Error executing lua: [string "<nvim>"]:0: arg1: expected ?, got 3',
matches('arg1: expected %?, got 3',
pcall_err(exec_lua, "vim.validate{arg1={3, function(a) return a == 1 end}}"))
-- Pass an additional message back.
eq('Error executing lua: [string "<nvim>"]:0: arg1: expected ?, got 3. Info: TEST_MSG',
matches('arg1: expected %?, got 3. Info: TEST_MSG',
pcall_err(exec_lua, "vim.validate{arg1={3, function(a) return a == 1, 'TEST_MSG' end}}"))
end)
@ -982,7 +982,7 @@ describe('lua stdlib', function()
]]
eq(NIL, funcs.luaeval "vim.g.to_delete")
matches([[^Error executing lua: .*: attempt to index .* nil value]],
matches([[attempt to index .* nil value]],
pcall_err(exec_lua, 'return vim.g[0].testing'))
end)
@ -1009,7 +1009,7 @@ describe('lua stdlib', function()
return {vim.b.nonexistant == vim.NIL, vim.b.nullvar == vim.NIL}
]])
matches([[^Error executing lua: .*: attempt to index .* nil value]],
matches([[attempt to index .* nil value]],
pcall_err(exec_lua, 'return vim.b[BUF][0].testing'))
eq({hello="world"}, funcs.luaeval "vim.b.to_delete")
@ -1046,7 +1046,7 @@ describe('lua stdlib', function()
eq(NIL, funcs.luaeval "vim.w.nonexistant")
eq(NIL, funcs.luaeval "vim.w[WIN].nonexistant")
matches([[^Error executing lua: .*: attempt to index .* nil value]],
matches([[attempt to index .* nil value]],
pcall_err(exec_lua, 'return vim.w[WIN][0].testing'))
eq({hello="world"}, funcs.luaeval "vim.w.to_delete")
@ -1078,7 +1078,7 @@ describe('lua stdlib', function()
eq(123, funcs.luaeval "vim.t[0].other")
eq(NIL, funcs.luaeval "vim.t[0].nonexistant")
matches([[^Error executing lua: .*: attempt to index .* nil value]],
matches([[attempt to index .* nil value]],
pcall_err(exec_lua, 'return vim.t[0][0].testing'))
eq({hello="world"}, funcs.luaeval "vim.t.to_delete")
@ -1108,7 +1108,7 @@ describe('lua stdlib', function()
eq(funcs.luaeval "vim.api.nvim_get_vvar('progpath')", funcs.luaeval "vim.v.progpath")
eq(false, funcs.luaeval "vim.v['false']")
eq(NIL, funcs.luaeval "vim.v.null")
matches([[^Error executing lua: .*: attempt to index .* nil value]],
matches([[attempt to index .* nil value]],
pcall_err(exec_lua, 'return vim.v[0].progpath'))
end)
@ -1128,9 +1128,9 @@ describe('lua stdlib', function()
]]
eq('', funcs.luaeval "vim.bo.filetype")
eq(true, funcs.luaeval "vim.bo[BUF].modifiable")
matches("^Error executing lua: .*: Invalid option name: 'nosuchopt'$",
matches("Invalid option name: 'nosuchopt'$",
pcall_err(exec_lua, 'return vim.bo.nosuchopt'))
matches("^Error executing lua: .*: Expected lua string$",
matches("Expected lua string$",
pcall_err(exec_lua, 'return vim.bo[0][0].autoread'))
end)
@ -1147,9 +1147,9 @@ describe('lua stdlib', function()
eq(0, funcs.luaeval "vim.wo.cole")
eq(0, funcs.luaeval "vim.wo[0].cole")
eq(0, funcs.luaeval "vim.wo[1001].cole")
matches("^Error executing lua: .*: Invalid option name: 'notanopt'$",
matches("Invalid option name: 'notanopt'$",
pcall_err(exec_lua, 'return vim.wo.notanopt'))
matches("^Error executing lua: .*: Expected lua string$",
matches("Expected lua string$",
pcall_err(exec_lua, 'return vim.wo[0][0].list'))
eq(2, funcs.luaeval "vim.wo[1000].cole")
exec_lua [[

View File

@ -20,6 +20,7 @@ local nvim_prog = helpers.nvim_prog
local nvim_set = helpers.nvim_set
local ok = helpers.ok
local read_file = helpers.read_file
local exec_lua = helpers.exec_lua
if helpers.pending_win32(pending) then return end
@ -580,21 +581,34 @@ describe('TUI', function()
end)
it("paste: 'nomodifiable' buffer", function()
local has_luajit = exec_lua('return jit ~= nil')
child_session:request('nvim_command', 'set nomodifiable')
child_session:request('nvim_exec_lua', [[
-- Stack traces for this test are non-deterministic, so disable them
_G.debug.traceback = function(msg) return msg end
]], {})
feed_data('\027[200~fail 1\nfail 2\n\027[201~')
screen:expect{grid=[[
|
{4:~ }|
{5: }|
{8:paste: Error executing lua: vim.lua:243: Vim:E21: }|
{8:Cannot make changes, 'modifiable' is off} |
{10:Press ENTER or type command to continue}{1: } |
{3:-- TERMINAL --} |
]]}
if has_luajit then
screen:expect{grid=[[
|
{4:~ }|
{5: }|
{8:paste: Error executing lua: vim.lua:0: Vim:E21: Ca}|
{8:nnot make changes, 'modifiable' is off} |
{10:Press ENTER or type command to continue}{1: } |
{3:-- TERMINAL --} |
]]}
else
screen:expect{grid=[[
|
{4:~ }|
{5: }|
{8:paste: Error executing lua: Vim:E21: Cannot make c}|
{8:hanges, 'modifiable' is off} |
{10:Press ENTER or type command to continue}{1: } |
{3:-- TERMINAL --} |
]]}
end
feed_data('\n') -- <Enter>
child_session:request('nvim_command', 'set modifiable')
feed_data('\027[200~success 1\nsuccess 2\n\027[201~')