From eb5d15e3838f53e2fcd25989c88db87458e9f984 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Sun, 7 Jan 2024 13:05:03 +0100 Subject: [PATCH] refactor: rewrite python provider in lua --- runtime/autoload/provider/python3.vim | 48 ++----- runtime/autoload/provider/pythonx.vim | 112 ---------------- runtime/autoload/remote/host.vim | 6 +- runtime/lua/nvim/health.lua | 4 - runtime/lua/provider/python/health.lua | 16 +-- runtime/lua/vim/provider/python.lua | 150 ++++++++++++++++++++++ test/functional/ex_cmds/script_spec.lua | 2 +- test/functional/helpers.lua | 9 +- test/functional/provider/python3_spec.lua | 2 +- test/old/testdir/test_python3.vim | 2 +- test/old/testdir/test_pyx3.vim | 2 +- 11 files changed, 175 insertions(+), 178 deletions(-) delete mode 100644 runtime/autoload/provider/pythonx.vim create mode 100644 runtime/lua/vim/provider/python.lua diff --git a/runtime/autoload/provider/python3.vim b/runtime/autoload/provider/python3.vim index 38ef0cccfc..d4ffeab155 100644 --- a/runtime/autoload/provider/python3.vim +++ b/runtime/autoload/provider/python3.vim @@ -1,45 +1,15 @@ -" The Python3 provider uses a Python3 host to emulate an environment for running -" python3 plugins. :help provider -" -" Associating the plugin with the Python3 host is the first step because -" plugins will be passed as command-line arguments - if exists('g:loaded_python3_provider') finish endif -let [s:prog, s:err] = provider#pythonx#Detect(3) -let g:loaded_python3_provider = empty(s:prog) ? 1 : 2 - -function! provider#python3#Prog() abort - return s:prog -endfunction - -function! provider#python3#Error() abort - return s:err -endfunction - -" The Python3 provider plugin will run in a separate instance of the Python3 -" host. -call remote#host#RegisterClone('legacy-python3-provider', 'python3') -call remote#host#RegisterPlugin('legacy-python3-provider', 'script_host.py', []) function! provider#python3#Call(method, args) abort - if s:err != '' - return - endif - if !exists('s:host') - let s:rpcrequest = function('rpcrequest') - - " Ensure that we can load the Python3 host before bootstrapping - try - let s:host = remote#host#Require('legacy-python3-provider') - catch - let s:err = v:exception - echohl WarningMsg - echomsg v:exception - echohl None - return - endtry - endif - return call(s:rpcrequest, insert(insert(a:args, 'python_'.a:method), s:host)) + return v:lua.require'vim.provider.python'.call(a:method, a:args) endfunction + +function! provider#python3#Require(host) abort + return v:lua.require'vim.provider.python'.require(a:host) +endfunction + +let s:prog = v:lua.require'vim.provider.python'.detect_by_module('neovim') +let g:loaded_python3_provider = empty(s:prog) ? 1 : 2 +call v:lua.require'vim.provider.python'.start() diff --git a/runtime/autoload/provider/pythonx.vim b/runtime/autoload/provider/pythonx.vim deleted file mode 100644 index 48b96c699a..0000000000 --- a/runtime/autoload/provider/pythonx.vim +++ /dev/null @@ -1,112 +0,0 @@ -" The Python provider helper -if exists('s:loaded_pythonx_provider') - finish -endif - -let s:loaded_pythonx_provider = 1 - -function! provider#pythonx#Require(host) abort - " Python host arguments - let prog = provider#python3#Prog() - let args = [prog, '-c', 'import sys; sys.path = [p for p in sys.path if p != ""]; import neovim; neovim.start_host()'] - - - " Collect registered Python plugins into args - let python_plugins = remote#host#PluginsForHost(a:host.name) - for plugin in python_plugins - call add(args, plugin.path) - endfor - - return provider#Poll(args, a:host.orig_name, '$NVIM_PYTHON_LOG_FILE', {'overlapped': v:true}) -endfunction - -function! s:get_python_executable_from_host_var(major_version) abort - return expand(get(g:, 'python'.(a:major_version == 3 ? '3' : execute("throw 'unsupported'")).'_host_prog', ''), v:true) -endfunction - -function! s:get_python_candidates(major_version) abort - return { - \ 3: ['python3', 'python3.12', 'python3.11', 'python3.10', 'python3.9', 'python3.8', 'python3.7', 'python'] - \ }[a:major_version] -endfunction - -" Returns [path_to_python_executable, error_message] -function! provider#pythonx#Detect(major_version) abort - return provider#pythonx#DetectByModule('neovim', a:major_version) -endfunction - -" Returns [path_to_python_executable, error_message] -function! provider#pythonx#DetectByModule(module, major_version) abort - let python_exe = s:get_python_executable_from_host_var(a:major_version) - - if !empty(python_exe) - return [exepath(expand(python_exe, v:true)), ''] - endif - - let candidates = s:get_python_candidates(a:major_version) - let errors = [] - - for exe in candidates - let [result, error] = provider#pythonx#CheckForModule(exe, a:module, a:major_version) - if result - return [exe, error] - endif - " Accumulate errors in case we don't find any suitable Python executable. - call add(errors, error) - endfor - - " No suitable Python executable found. - return ['', 'Could not load Python '.a:major_version.":\n".join(errors, "\n")] -endfunction - -" Returns array: [prog_exitcode, prog_version] -function! s:import_module(prog, module) abort - let prog_version = system([a:prog, '-W', 'ignore', '-c', printf( - \ 'import sys, importlib.util; ' . - \ 'sys.path = [p for p in sys.path if p != ""]; ' . - \ 'sys.stdout.write(str(sys.version_info[0]) + "." + str(sys.version_info[1])); ' . - \ 'sys.exit(2 * int(importlib.util.find_spec("%s") is None))', - \ a:module)]) - return [v:shell_error, prog_version] -endfunction - -" Returns array: [was_success, error_message] -function! provider#pythonx#CheckForModule(prog, module, major_version) abort - let prog_path = exepath(a:prog) - if prog_path ==# '' - return [0, a:prog . ' not found in search path or not executable.'] - endif - - let min_version = '3.7' - - " Try to load module, and output Python version. - " Exit codes: - " 0 module can be loaded. - " 2 module cannot be loaded. - " Otherwise something else went wrong (e.g. 1 or 127). - let [prog_exitcode, prog_version] = s:import_module(a:prog, a:module) - - if prog_exitcode == 2 || prog_exitcode == 0 - " Check version only for expected return codes. - if prog_version !~ '^' . a:major_version - return [0, prog_path . ' is Python ' . prog_version . ' and cannot provide Python ' - \ . a:major_version . '.'] - elseif prog_version =~ '^' . a:major_version && str2nr(prog_version[2:]) < str2nr(min_version[2:]) - return [0, prog_path . ' is Python ' . prog_version . ' and cannot provide Python >= ' - \ . min_version . '.'] - endif - endif - - if prog_exitcode == 2 - return [0, prog_path.' does not have the "' . a:module . '" module.'] - elseif prog_exitcode == 127 - " This can happen with pyenv's shims. - return [0, prog_path . ' does not exist: ' . prog_version] - elseif prog_exitcode - return [0, 'Checking ' . prog_path . ' caused an unknown error. ' - \ . '(' . prog_exitcode . ', output: ' . prog_version . ')' - \ . ' Report this at https://github.com/neovim/neovim'] - endif - - return [1, ''] -endfunction diff --git a/runtime/autoload/remote/host.vim b/runtime/autoload/remote/host.vim index 884b478f19..0032a4b71e 100644 --- a/runtime/autoload/remote/host.vim +++ b/runtime/autoload/remote/host.vim @@ -190,11 +190,9 @@ endfunction " Registration of standard hosts -" Python/Python3 -call remote#host#Register('python', '*', - \ function('provider#pythonx#Require')) +" Python3 call remote#host#Register('python3', '*', - \ function('provider#pythonx#Require')) + \ function('provider#python3#Require')) " Ruby call remote#host#Register('ruby', '*.rb', diff --git a/runtime/lua/nvim/health.lua b/runtime/lua/nvim/health.lua index 5a643db0ba..1a440c9827 100644 --- a/runtime/lua/nvim/health.lua +++ b/runtime/lua/nvim/health.lua @@ -183,10 +183,6 @@ local function check_rplugin_manifest() health.start('Remote Plugins') local existing_rplugins = {} - for _, item in ipairs(vim.fn['remote#host#PluginsForHost']('python')) do - existing_rplugins[item.path] = 'python' - end - for _, item in ipairs(vim.fn['remote#host#PluginsForHost']('python3')) do existing_rplugins[item.path] = 'python3' end diff --git a/runtime/lua/provider/python/health.lua b/runtime/lua/provider/python/health.lua index 6d3a4d5c50..825fddc917 100644 --- a/runtime/lua/provider/python/health.lua +++ b/runtime/lua/provider/python/health.lua @@ -217,7 +217,7 @@ end function M.check() health.start('Python 3 provider (optional)') - local pyname = 'python3' + local pyname = 'python3' ---@type string? local python_exe = '' local virtual_env = os.getenv('VIRTUAL_ENV') local venv = virtual_env and vim.fn.resolve(virtual_env) or '' @@ -237,11 +237,10 @@ function M.check() health.info(message) end - local python_table = vim.fn['provider#pythonx#Detect'](3) - pyname = python_table[1] - local pythonx_warnings = python_table[2] + local pythonx_warnings + pyname, pythonx_warnings = require('vim.provider.python').detect_by_module('neovim') - if pyname == '' then + if not pyname then health.warn( 'No Python executable found that can `import neovim`. ' .. 'Using the first available executable for diagnostics.' @@ -251,7 +250,7 @@ function M.check() end -- No Python executable could `import neovim`, or host_prog_var was used. - if pythonx_warnings ~= '' then + if pythonx_warnings then health.warn(pythonx_warnings, { 'See :help provider-python for more information.', 'You may disable this provider (and warning) by adding `let g:loaded_python3_provider = 0` to your init.vim', @@ -364,9 +363,8 @@ function M.check() -- can import 'pynvim'. If so, that Python failed to import 'neovim' as -- well, which is most probably due to a failed pip upgrade: -- https://github.com/neovim/neovim/wiki/Following-HEAD#20181118 - local pynvim_table = vim.fn['provider#pythonx#DetectByModule']('pynvim', 3) - local pynvim_exe = pynvim_table[1] - if pynvim_exe ~= '' then + local pynvim_exe = require('vim.provider.python').detect_by_module('pynvim') + if pynvim_exe then local message = 'Detected pip upgrade failure: Python executable can import "pynvim" but not "neovim": ' .. pynvim_exe local advice = { diff --git a/runtime/lua/vim/provider/python.lua b/runtime/lua/vim/provider/python.lua new file mode 100644 index 0000000000..94872437db --- /dev/null +++ b/runtime/lua/vim/provider/python.lua @@ -0,0 +1,150 @@ +local M = {} +local min_version = '3.7' +local s_err ---@type string? +local s_host ---@type string? + +local python_candidates = { + 'python3', + 'python3.12', + 'python3.11', + 'python3.10', + 'python3.9', + 'python3.8', + 'python3.7', + 'python', +} + +--- @param prog string +--- @param module string +--- @return integer, string +local function import_module(prog, module) + local program = [[ +import sys, importlib.util; +sys.path = [p for p in sys.path if p != ""]; +sys.stdout.write(str(sys.version_info[0]) + "." + str(sys.version_info[1]));]] + + program = program + .. string.format('sys.exit(2 * int(importlib.util.find_spec("%s") is None))', module) + + local out = vim.system({ prog, '-W', 'ignore', '-c', program }):wait() + return out.code, assert(out.stdout) +end + +--- @param prog string +--- @param module string +--- @return string? +local function check_for_module(prog, module) + local prog_path = vim.fn.exepath(prog) + if prog_path == '' then + return prog .. ' not found in search path or not executable.' + end + + -- Try to load module, and output Python version. + -- Exit codes: + -- 0 module can be loaded. + -- 2 module cannot be loaded. + -- Otherwise something else went wrong (e.g. 1 or 127). + local prog_exitcode, prog_version = import_module(prog, module) + if prog_exitcode == 2 or prog_exitcode == 0 then + -- Check version only for expected return codes. + if vim.version.lt(prog_version, min_version) then + return string.format( + '%s is Python %s and cannot provide Python >= %s.', + prog_path, + prog_version, + min_version + ) + end + end + + if prog_exitcode == 2 then + return string.format('%s does not have the "%s" module.', prog_path, module) + elseif prog_exitcode == 127 then + -- This can happen with pyenv's shims. + return string.format('%s does not exist: %s', prog_path, prog_version) + elseif prog_exitcode ~= 0 then + return string.format( + 'Checking %s caused an unknown error. (%s, output: %s) Report this at https://github.com/neovim/neovim', + prog_path, + prog_exitcode, + prog_version + ) + end + + return nil +end + +--- @param module string +--- @return string? path to detected python, if any; nil if not found +--- @return string? error message if python can't be detected by {module}; nil if success +function M.detect_by_module(module) + local python_exe = vim.fn.expand(vim.g.python3_host_prog or '', true) + + if python_exe ~= '' then + return vim.fn.exepath(vim.fn.expand(python_exe, true)), nil + end + + local errors = {} + for _, exe in ipairs(python_candidates) do + local error = check_for_module(exe, module) + if not error then + return exe, error + end + -- Accumulate errors in case we don't find any suitable Python executable. + table.insert(errors, error) + end + + -- No suitable Python executable found. + return nil, 'Could not load Python :\n' .. table.concat(errors, '\n') +end + +function M.require(host) + -- Python host arguments + local prog = M.detect_by_module('neovim') + local args = { + prog, + '-c', + 'import sys; sys.path = [p for p in sys.path if p != ""]; import neovim; neovim.start_host()', + } + + -- Collect registered Python plugins into args + local python_plugins = vim.fn['remote#host#PluginsForHost'](host.name) ---@type any + ---@param plugin any + for _, plugin in ipairs(python_plugins) do + table.insert(args, plugin.path) + end + + return vim.fn['provider#Poll']( + args, + host.orig_name, + '$NVIM_PYTHON_LOG_FILE', + { ['overlapped'] = true } + ) +end + +function M.call(method, args) + if s_err then + return + end + + if not s_host then + -- Ensure that we can load the Python3 host before bootstrapping + local ok, result = pcall(vim.fn['remote#host#Require'], 'legacy-python3-provider') ---@type any, any + if not ok then + s_err = result + vim.api.nvim_echo({ result, 'WarningMsg' }, true, {}) + return + end + s_host = result + end + + return vim.fn.rpcrequest(s_host, 'python_' .. method, unpack(args)) +end + +function M.start() + -- The Python3 provider plugin will run in a separate instance of the Python3 host. + vim.fn['remote#host#RegisterClone']('legacy-python3-provider', 'python3') + vim.fn['remote#host#RegisterPlugin']('legacy-python3-provider', 'script_host.py', {}) +end + +return M diff --git a/test/functional/ex_cmds/script_spec.lua b/test/functional/ex_cmds/script_spec.lua index ebdaa0f656..4c963c5da7 100644 --- a/test/functional/ex_cmds/script_spec.lua +++ b/test/functional/ex_cmds/script_spec.lua @@ -96,7 +96,7 @@ describe('script_get-based command', function() -- Provider-based scripts test_garbage_exec('ruby', not missing_provider('ruby')) - test_garbage_exec('python3', not missing_provider('python3')) + test_garbage_exec('python3', not missing_provider('python')) -- Missing scripts test_garbage_exec('python', false) diff --git a/test/functional/helpers.lua b/test/functional/helpers.lua index eddf336b6f..159016b484 100644 --- a/test/functional/helpers.lua +++ b/test/functional/helpers.lua @@ -932,17 +932,14 @@ function module.new_pipename() end --- @param provider string ---- @return string|false? +--- @return string|boolean? function module.missing_provider(provider) if provider == 'ruby' or provider == 'node' or provider == 'perl' then --- @type string? local e = module.fn['provider#' .. provider .. '#Detect']()[2] return e ~= '' and e or false - elseif provider == 'python' or provider == 'python3' then - local py_major_version = (provider == 'python3' and 3 or 2) - --- @type string? - local e = module.fn['provider#pythonx#Detect'](py_major_version)[2] - return e ~= '' and e or false + elseif provider == 'python' then + return module.exec_lua([[return {require('vim.provider.python').detect_by_module('neovim')}]])[2] end assert(false, 'Unknown provider: ' .. provider) end diff --git a/test/functional/provider/python3_spec.lua b/test/functional/provider/python3_spec.lua index 1419d7f651..9bde57f777 100644 --- a/test/functional/provider/python3_spec.lua +++ b/test/functional/provider/python3_spec.lua @@ -13,7 +13,7 @@ local dedent = helpers.dedent do clear() - local reason = missing_provider('python3') + local reason = missing_provider('python') if reason then it(':python3 reports E319 if provider is missing', function() local expected = [[Vim%(py3.*%):E319: No "python3" provider found.*]] diff --git a/test/old/testdir/test_python3.vim b/test/old/testdir/test_python3.vim index 23c63f22d8..c9dbc0b84e 100644 --- a/test/old/testdir/test_python3.vim +++ b/test/old/testdir/test_python3.vim @@ -169,7 +169,7 @@ func Test_Catch_Exception_Message() try py3 raise RuntimeError( 'TEST' ) catch /.*/ - call assert_match('^Vim(.*):.*RuntimeError: TEST$', v:exception ) + call assert_match('^Vim(.*):.*RuntimeError: TEST.*$', v:exception ) endtry endfunc diff --git a/test/old/testdir/test_pyx3.vim b/test/old/testdir/test_pyx3.vim index 09ece6f812..89a3cc22ff 100644 --- a/test/old/testdir/test_pyx3.vim +++ b/test/old/testdir/test_pyx3.vim @@ -76,7 +76,7 @@ func Test_Catch_Exception_Message() try pyx raise RuntimeError( 'TEST' ) catch /.*/ - call assert_match('^Vim(.*):.*RuntimeError: TEST$', v:exception ) + call assert_match('^Vim(.*):.*RuntimeError: TEST.*$', v:exception ) endtry endfunc