fix(vim.ui.open): return (don't show) error message

Problem:
Showing an error via vim.notify() makes it awkward for callers such as
lsp/handlers.lua to avoid showing redundant errors.

Solution:
Return the message instead of showing it. Let the caller decide whether
and when to show the message.

runtime/doc/lua.txt

@@ -2345,7 +2345,8 @@ input({opts}, {on_confirm})                                   *vim.ui.input()*
 
 open({path})                                                   *vim.ui.open()*
     Opens `path` with the system default handler (macOS `open`, Windows
-    `explorer.exe`, Linux `xdg-open`, …), or shows a message on failure.
+    `explorer.exe`, Linux `xdg-open`, …), or returns (but does not show) an
+    error message on failure.
 
     Expands "~/" and environment variables in filesystem paths.
 
@@ -2360,7 +2361,8 @@ open({path})                                                   *vim.ui.open()*
       • {path}  (string) Path or URL to open
 
     Return: ~
-        SystemCompleted|nil result Command result, or nil if not found.
+        SystemCompleted|nil # Command result, or nil if not found.
+        (string|nil) # Error message on failure
 
     See also: ~
       • |vim.system()|

runtime/lua/vim/lsp/handlers.lua

@@ -573,15 +573,14 @@ M['window/showDocument'] = function(_, result, ctx, _)
 
   if result.external then
     -- TODO(lvimuser): ask the user for confirmation
-
-    local ret = vim.ui.open(uri)
+    local ret, err = vim.ui.open(uri)
 
     if ret == nil or ret.code ~= 0 then
       return {
         success = false,
         error = {
           code = protocol.ErrorCodes.UnknownErrorCode,
-          message = ret and ret.stderr or 'No handler found',
+          message = ret and ret.stderr or err,
         },
       }
     end

runtime/lua/vim/ui.lua

@@ -105,7 +105,7 @@ function M.input(opts, on_confirm)
 end
 
 --- Opens `path` with the system default handler (macOS `open`, Windows `explorer.exe`, Linux
---- `xdg-open`, …), or shows a message on failure.
+--- `xdg-open`, …), or returns (but does not show) an error message on failure.
 ---
 --- Expands "~/" and environment variables in filesystem paths.
 ---
@@ -118,13 +118,14 @@ end
 ---
 ---@param path string Path or URL to open
 ---
----@return SystemCompleted|nil result Command result, or nil if not found.
+---@return SystemCompleted|nil # Command result, or nil if not found.
+---@return string|nil # Error message on failure
 ---
 ---@see |vim.system()|
 function M.open(path)
-  vim.validate{
-    path={path, 'string'}
-  }
+  vim.validate({
+    path = { path, 'string' },
+  })
   local is_uri = path:match('%w+:')
   if not is_uri then
     path = vim.fn.expand(path)
@@ -141,17 +142,16 @@ function M.open(path)
   elseif vim.fn.executable('xdg-open') == 1 then
     cmd = { 'xdg-open', path }
   else
-    vim.notify('vim.ui.open: no handler found (tried: wslview, xdg-open)', vim.log.levels.ERROR)
-    return nil
+    return nil, 'vim.ui.open: no handler found (tried: wslview, xdg-open)'
   end
 
-  local rv = vim.system(cmd, { text = true, detach = true, }):wait()
+  local rv = vim.system(cmd, { text = true, detach = true }):wait()
   if rv.code ~= 0 then
     local msg = ('vim.ui.open: command failed (%d): %s'):format(rv.code, vim.inspect(cmd))
-    vim.notify(msg, vim.log.levels.ERROR)
+    return rv, msg
   end
 
-  return rv
+  return rv, nil
 end
 
 return M

runtime/plugin/nvim.lua

@@ -22,12 +22,23 @@ end, { desc = 'Inspect treesitter language tree for buffer', count = true })
 -- TODO: use vim.region() when it lands... #13896 #16843
 local function get_visual_selection()
   local save_a = vim.fn.getreginfo('a')
-  vim.cmd[[norm! "ay]]
+  vim.cmd([[norm! "ay]])
   local selection = vim.fn.getreg('a', 1)
   vim.fn.setreg('a', save_a)
   return selection
 end
 
-local gx_desc = 'Opens filepath or URI under cursor with the system handler (file explorer, web browser, …)'
-vim.keymap.set({ 'n' }, 'gx', function() vim.ui.open(vim.fn.expand('<cfile>')) end, { desc = gx_desc })
-vim.keymap.set({ 'x' }, 'gx', function() vim.ui.open(get_visual_selection()) end, { desc = gx_desc })
+local gx_desc =
+  'Opens filepath or URI under cursor with the system handler (file explorer, web browser, …)'
+local function do_open(uri)
+  local _, err = vim.ui.open(uri)
+  if err then
+    vim.notify(err, vim.log.levels.ERROR)
+  end
+end
+vim.keymap.set({ 'n' }, 'gx', function()
+  do_open(vim.fn.expand('<cfile>'))
+end, { desc = gx_desc })
+vim.keymap.set({ 'x' }, 'gx', function()
+  do_open(get_visual_selection())
+end, { desc = gx_desc })

test/functional/lua/ui_spec.lua

@@ -5,6 +5,7 @@ local exec_lua = helpers.exec_lua
 local clear = helpers.clear
 local feed = helpers.feed
 local eval = helpers.eval
+local is_os = helpers.is_os
 local poke_eventloop = helpers.poke_eventloop
 
 describe('vim.ui', function()
@@ -133,15 +134,17 @@ describe('vim.ui', function()
 
   describe('open()', function()
     it('validation', function()
-      exec_lua[[vim.ui.open('non-existent-file')]]
-      matches('vim.ui.open: command failed %(%d%): { "[^"]+", "non%-existent%-file" }', eval('v:errmsg'))
+      if not is_os('bsd') then
+        matches('vim.ui.open: command failed %(%d%): { "[^"]+", "non%-existent%-file" }',
+          exec_lua[[local _, err = vim.ui.open('non-existent-file') ; return err]])
+      end
 
       exec_lua[[
         vim.fn.has = function() return 0 end
         vim.fn.executable = function() return 0 end
       ]]
-      exec_lua[[vim.ui.open('foo')]]
-      eq('vim.ui.open: no handler found (tried: wslview, xdg-open)', eval('v:errmsg'))
+      eq('vim.ui.open: no handler found (tried: wslview, xdg-open)',
+        exec_lua[[local _, err = vim.ui.open('foo') ; return err]])
     end)
   end)
 end)