wyj/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2024-09-17 20:58:20 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(shada): rework msgpack decoding without msgpack-c

Browse Source 
This also makes shada reading slightly faster due to avoiding
some copying and allocation.

Use keysets to drive decoding of msgpack maps for shada entries.
This commit is contained in:
bfredl 2024-07-02 13:45:50 +02:00
parent 0c2860d9e5
commit f926cc32c9
42 changed files with 1025 additions and 1165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/gen_eval_files.lua
View File

@ -376,6 +376,9 @@ end
--- @param fun vim.EvalFn
--- @param write fun(line: string)
local function render_api_keyset_meta(_f, fun, write)
if string.sub(fun.name, 1, 1) == '_' then
return -- not exported
end
write('')
write('--- @class vim.api.keyset.' .. fun.name)
for _, p in ipairs(fun.params) do

6
src/nvim/api/command.c
View File

@ -515,7 +515,7 @@ String nvim_cmd(uint64_t channel_id, Dict(cmd) *cmd, Dict(cmd_opts) *opts, Arena
if (HAS_KEY(cmd, cmd, magic)) {
Dict(cmd_magic) magic[1] = KEYDICT_INIT;
if (!api_dict_to_keydict(magic, KeyDict_cmd_magic_get_field, cmd->magic, err)) {
if (!api_dict_to_keydict(magic, DictHash(cmd_magic), cmd->magic, err)) {
goto end;
}
@ -533,14 +533,14 @@ String nvim_cmd(uint64_t channel_id, Dict(cmd) *cmd, Dict(cmd_opts) *opts, Arena
if (HAS_KEY(cmd, cmd, mods)) {
Dict(cmd_mods) mods[1] = KEYDICT_INIT;
if (!api_dict_to_keydict(mods, KeyDict_cmd_mods_get_field, cmd->mods, err)) {
if (!api_dict_to_keydict(mods, DictHash(cmd_mods), cmd->mods, err)) {
goto end;
}
if (HAS_KEY(mods, cmd_mods, filter)) {
Dict(cmd_mods_filter) filter[1] = KEYDICT_INIT;
if (!api_dict_to_keydict(&filter, KeyDict_cmd_mods_filter_get_field,
if (!api_dict_to_keydict(&filter, DictHash(cmd_mods_filter),
mods->filter, err)) {
goto end;
}

42
src/nvim/api/keysets_defs.h
View File

@ -103,7 +103,7 @@ typedef struct {
Object nargs;
Object preview;
Object range;
Boolean register_;
Boolean register_ DictKey(register);
} Dict(user_command);
typedef struct {
@ -170,7 +170,7 @@ typedef struct {
Boolean reverse;
Boolean altfont;
Boolean nocombine;
Boolean default_;
Boolean default_ DictKey(default);
Object cterm;
Object foreground;
Object fg;
@ -392,3 +392,41 @@ typedef struct {
OptionalKeys is_set__ns_opts_;
Array wins;
} Dict(ns_opts);
typedef struct {
OptionalKeys is_set___shada_search_pat_;
Boolean magic DictKey(sm);
Boolean smartcase DictKey(sc);
Boolean has_line_offset DictKey(sl);
Boolean place_cursor_at_end DictKey(se);
Boolean is_last_used DictKey(su);
Boolean is_substitute_pattern DictKey(ss);
Boolean highlighted DictKey(sh);
Boolean search_backward DictKey(sb);
Integer offset DictKey(so);
String pat DictKey(sp);
} Dict(_shada_search_pat);
typedef struct {
OptionalKeys is_set___shada_mark_;
Integer n;
Integer l;
Integer c;
String f;
} Dict(_shada_mark);
typedef struct {
OptionalKeys is_set___shada_register_;
StringArray rc;
Boolean ru;
Integer rt;
Integer n;
Integer rw;
} Dict(_shada_register);
typedef struct {
OptionalKeys is_set___shada_buflist_item_;
Integer l;
Integer c;
String f;
} Dict(_shada_buflist_item);

10
src/nvim/api/private/defs.h
View File

@ -19,6 +19,8 @@
# define ArrayOf(...) Array
# define DictionaryOf(...) Dictionary
# define Dict(name) KeyDict_##name
# define DictHash(name) KeyDict_##name##_get_field
# define DictKey(name)
# include "api/private/defs.h.inline.generated.h"
#endif
@ -88,6 +90,8 @@ typedef kvec_t(Object) Array;
typedef struct key_value_pair KeyValuePair;
typedef kvec_t(KeyValuePair) Dictionary;
typedef kvec_t(String) StringArray;
typedef enum {
kObjectTypeNil = 0,
kObjectTypeBoolean,
@ -103,6 +107,10 @@ typedef enum {
kObjectTypeTabpage,
} ObjectType;
typedef enum {
kUnpackTypeStringArray = -1,
} UnpackType;
/// Value by which objects represented as EXT type are shifted
///
/// Subtracted when packing, added when unpacking. Used to allow moving
@ -140,7 +148,7 @@ typedef struct {
typedef struct {
char *str;
size_t ptr_off;
ObjectType type; // kObjectTypeNil == untyped
int type; // ObjectType or UnpackType. kObjectTypeNil == untyped
int opt_index;
bool is_hlgroup;
} KeySetLink;

11
src/nvim/api/private/helpers.c
View File

@ -1,6 +1,5 @@
#include <assert.h>
#include <limits.h>
#include <msgpack/unpack.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stddef.h>
@ -922,15 +921,17 @@ bool api_dict_to_keydict(void *retval, FieldHashfn hashy, Dictionary dict, Error
} else if (value->type == kObjectTypeDictionary) {
*val = value->data.dictionary;
} else {
api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
api_err_exp(err, field->str, api_typename((ObjectType)field->type),
api_typename(value->type));
return false;
}
} else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
|| field->type == kObjectTypeTabpage) {
if (value->type == kObjectTypeInteger || value->type == field->type) {
if (value->type == kObjectTypeInteger || value->type == (ObjectType)field->type) {
*(handle_T *)mem = (handle_T)value->data.integer;
} else {
api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
api_err_exp(err, field->str, api_typename((ObjectType)field->type),
api_typename(value->type));
return false;
}
} else if (field->type == kObjectTypeLuaRef) {
@ -980,7 +981,7 @@ Dictionary api_keydict_to_dict(void *value, KeySetLink *table, size_t max_size,
} else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
|| field->type == kObjectTypeTabpage) {
val.data.integer = *(handle_T *)mem;
val.type = field->type;
val.type = (ObjectType)field->type;
} else if (field->type == kObjectTypeLuaRef) {
// do nothing
} else {

1
src/nvim/api/ui.c
View File

@ -1,6 +1,5 @@
#include <assert.h>
#include <inttypes.h>
#include <msgpack/pack.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

2
src/nvim/buffer.c
View File

@ -868,7 +868,7 @@ static void free_buffer(buf_T *buf)
}
unref_var_dict(buf->b_vars);
aubuflocal_remove(buf);
tv_dict_unref(buf->additional_data);
xfree(buf->additional_data);
xfree(buf->b_prompt_text);
callback_free(&buf->b_prompt_callback);
callback_free(&buf->b_prompt_interrupt);

2
src/nvim/buffer_defs.h
View File

@ -712,7 +712,7 @@ struct file_buffer {
Terminal *terminal; // Terminal instance associated with the buffer
dict_T *additional_data; // Additional data from shada file if any.
AdditionalData *additional_data; // Additional data from shada file if any.
int b_mapped_ctrl_c; // modes where CTRL-C is mapped

10
src/nvim/cmdhist.c
View File

@ -190,7 +190,7 @@ static inline void hist_free_entry(histentry_T *hisptr)
FUNC_ATTR_NONNULL_ALL
{
xfree(hisptr->hisstr);
tv_list_unref(hisptr->additional_elements);
xfree(hisptr->additional_data);
clear_hist_entry(hisptr);
}
@ -237,7 +237,7 @@ static int in_history(int type, const char *str, int move_to_front, int sep)
return false;
}
list_T *const list = history[type][i].additional_elements;
AdditionalData *ad = history[type][i].additional_data;
char *const save_hisstr = history[type][i].hisstr;
while (i != hisidx[type]) {
if (++i >= hislen) {
@ -246,11 +246,11 @@ static int in_history(int type, const char *str, int move_to_front, int sep)
history[type][last_i] = history[type][i];
last_i = i;
}
tv_list_unref(list);
xfree(ad);
history[type][i].hisnum = ++hisnum[type];
history[type][i].hisstr = save_hisstr;
history[type][i].timestamp = os_time();
history[type][i].additional_elements = NULL;
history[type][i].additional_data = NULL;
return true;
}
@ -337,7 +337,7 @@ void add_to_history(int histype, const char *new_entry, size_t new_entrylen, boo
// Store the separator after the NUL of the string.
hisptr->hisstr = xstrnsave(new_entry, new_entrylen + 2);
hisptr->timestamp = os_time();
hisptr->additional_elements = NULL;
hisptr->additional_data = NULL;
hisptr->hisstr[new_entrylen + 1] = (char)sep;
hisptr->hisnum = ++hisnum[histype];

2
src/nvim/cmdhist.h
View File

@ -24,7 +24,7 @@ typedef struct {
int hisnum; ///< Entry identifier number.
char *hisstr; ///< Actual entry, separator char after the NUL.
Timestamp timestamp; ///< Time when entry was added.
list_T *additional_elements; ///< Additional entries from ShaDa file.
AdditionalData *additional_data; ///< Additional entries from ShaDa file.
} histentry_T;
#ifdef INCLUDE_GENERATED_DECLARATIONS

1
src/nvim/context.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <msgpack/sbuffer.h>
#include <stddef.h>
#include "klib/kvec.h"

99
src/nvim/eval.c
View File

@ -4788,19 +4788,6 @@ bool garbage_collect(bool testing)
FOR_ALL_BUFFERS(buf) {
// buffer-local variables
ABORTING(set_ref_in_item)(&buf->b_bufvar.di_tv, copyID, NULL, NULL);
// buffer marks (ShaDa additional data)
ABORTING(set_ref_in_fmark)(buf->b_last_cursor, copyID);
ABORTING(set_ref_in_fmark)(buf->b_last_insert, copyID);
ABORTING(set_ref_in_fmark)(buf->b_last_change, copyID);
for (size_t i = 0; i < NMARKS; i++) {
ABORTING(set_ref_in_fmark)(buf->b_namedm[i], copyID);
}
// buffer change list (ShaDa additional data)
for (int i = 0; i < buf->b_changelistlen; i++) {
ABORTING(set_ref_in_fmark)(buf->b_changelist[i], copyID);
}
// buffer ShaDa additional data
ABORTING(set_ref_dict)(buf->additional_data, copyID);
// buffer callback functions
ABORTING(set_ref_in_callback)(&buf->b_prompt_callback, copyID, NULL, NULL);
@ -4823,10 +4810,6 @@ bool garbage_collect(bool testing)
FOR_ALL_TAB_WINDOWS(tp, wp) {
// window-local variables
ABORTING(set_ref_in_item)(&wp->w_winvar.di_tv, copyID, NULL, NULL);
// window jump list (ShaDa additional data)
for (int i = 0; i < wp->w_jumplistlen; i++) {
ABORTING(set_ref_in_fmark)(wp->w_jumplist[i].fmark, copyID);
}
}
// window-local variables in autocmd windows
for (int i = 0; i < AUCMD_WIN_COUNT; i++) {
@ -4843,9 +4826,6 @@ bool garbage_collect(bool testing)
char name = NUL;
bool is_unnamed = false;
reg_iter = op_global_reg_iter(reg_iter, &name, &reg, &is_unnamed);
if (name != NUL) {
ABORTING(set_ref_dict)(reg.additional_data, copyID);
}
} while (reg_iter != NULL);
}
@ -4856,9 +4836,6 @@ bool garbage_collect(bool testing)
xfmark_T fm;
char name = NUL;
mark_iter = mark_global_iter(mark_iter, &name, &fm);
if (name != NUL) {
ABORTING(set_ref_dict)(fm.fmark.additional_data, copyID);
}
} while (mark_iter != NULL);
}
@ -4900,36 +4877,6 @@ bool garbage_collect(bool testing)
// v: vars
ABORTING(set_ref_in_ht)(&vimvarht, copyID, NULL);
// history items (ShaDa additional elements)
if (p_hi) {
for (int i = 0; i < HIST_COUNT; i++) {
const void *iter = NULL;
do {
histentry_T hist;
iter = hist_iter(iter, (uint8_t)i, false, &hist);
if (hist.hisstr != NULL) {
ABORTING(set_ref_list)(hist.additional_elements, copyID);
}
} while (iter != NULL);
}
}
// previously used search/substitute patterns (ShaDa additional data)
{
SearchPattern pat;
get_search_pattern(&pat);
ABORTING(set_ref_dict)(pat.additional_data, copyID);
get_substitute_pattern(&pat);
ABORTING(set_ref_dict)(pat.additional_data, copyID);
}
// previously used replacement string
{
SubReplacementString sub;
sub_get_replacement(&sub);
ABORTING(set_ref_list)(sub.additional_elements, copyID);
}
ABORTING(set_ref_in_quickfix)(copyID);
bool did_free = false;
@ -5211,52 +5158,6 @@ bool set_ref_in_item(typval_T *tv, int copyID, ht_stack_T **ht_stack, list_stack
return abort;
}
/// Mark all lists and dicts referenced in given mark
///
/// @return true if setting references failed somehow.
static inline bool set_ref_in_fmark(fmark_T fm, int copyID)
FUNC_ATTR_WARN_UNUSED_RESULT
{
if (fm.additional_data != NULL
&& fm.additional_data->dv_copyID != copyID) {
fm.additional_data->dv_copyID = copyID;
return set_ref_in_ht(&fm.additional_data->dv_hashtab, copyID, NULL);
}
return false;
}
/// Mark all lists and dicts referenced in given list and the list itself
///
/// @return true if setting references failed somehow.
static inline bool set_ref_list(list_T *list, int copyID)
FUNC_ATTR_WARN_UNUSED_RESULT
{
if (list != NULL) {
typval_T tv = (typval_T) {
.v_type = VAR_LIST,
.vval = { .v_list = list }
};
return set_ref_in_item(&tv, copyID, NULL, NULL);
}
return false;
}
/// Mark all lists and dicts referenced in given dict and the dict itself
///
/// @return true if setting references failed somehow.
static inline bool set_ref_dict(dict_T *dict, int copyID)
FUNC_ATTR_WARN_UNUSED_RESULT
{
if (dict != NULL) {
typval_T tv = (typval_T) {
.v_type = VAR_DICT,
.vval = { .v_dict = dict }
};
return set_ref_in_item(&tv, copyID, NULL, NULL);
}
return false;
}
/// Get the key for #{key: val} into "tv" and advance "arg".
///
/// @return FAIL when there is no valid key.

321
src/nvim/eval/decode.c
View File

@ -1,5 +1,4 @@
#include <assert.h>
#include <msgpack/object.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
@ -7,6 +6,7 @@
#include <string.h>
#include "klib/kvec.h"
#include "mpack/object.h"
#include "nvim/ascii_defs.h"
#include "nvim/charset.h"
#include "nvim/eval.h"
@ -885,173 +885,250 @@ json_decode_string_ret:
#undef DICT_LEN
/// Convert msgpack object to a Vimscript one
int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv)
FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
static void positive_integer_to_special_typval(typval_T *rettv, uint64_t val)
{
switch (mobj.type) {
case MSGPACK_OBJECT_NIL:
if (val <= VARNUMBER_MAX) {
*rettv = (typval_T) {
.v_type = VAR_NUMBER,
.v_lock = VAR_UNLOCKED,
.vval = { .v_number = (varnumber_T)val },
};
} else {
list_T *const list = tv_list_alloc(4);
tv_list_ref(list);
create_special_dict(rettv, kMPInteger, ((typval_T) {
.v_type = VAR_LIST,
.v_lock = VAR_UNLOCKED,
.vval = { .v_list = list },
}));
tv_list_append_number(list, 1);
tv_list_append_number(list, (varnumber_T)((val >> 62) & 0x3));
tv_list_append_number(list, (varnumber_T)((val >> 31) & 0x7FFFFFFF));
tv_list_append_number(list, (varnumber_T)(val & 0x7FFFFFFF));
}
}
static void typval_parse_enter(mpack_parser_t *parser, mpack_node_t *node)
{
typval_T *result = NULL;
mpack_node_t *parent = MPACK_PARENT_NODE(node);
if (parent) {
switch (parent->tok.type) {
case MPACK_TOKEN_ARRAY: {
list_T *list = parent->data[1].p;
result = tv_list_append_owned_tv(list, (typval_T) { .v_type = VAR_UNKNOWN });
break;
}
case MPACK_TOKEN_MAP: {
typval_T(*items)[2] = parent->data[1].p;
result = &items[parent->pos][parent->key_visited];
break;
}
case MPACK_TOKEN_STR:
case MPACK_TOKEN_BIN:
case MPACK_TOKEN_EXT:
assert(node->tok.type == MPACK_TOKEN_CHUNK);
break;
default:
abort();
}
} else {
result = parser->data.p;
}
// for types that are completed in typval_parse_exit
node->data[0].p = result;
node->data[1].p = NULL; // free on error if non-NULL
switch (node->tok.type) {
case MPACK_TOKEN_NIL:
*result = (typval_T) {
.v_type = VAR_SPECIAL,
.v_lock = VAR_UNLOCKED,
.vval = { .v_special = kSpecialVarNull },
};
break;
case MSGPACK_OBJECT_BOOLEAN:
*rettv = (typval_T) {
case MPACK_TOKEN_BOOLEAN:
*result = (typval_T) {
.v_type = VAR_BOOL,
.v_lock = VAR_UNLOCKED,
.vval = {
.v_bool = mobj.via.boolean ? kBoolVarTrue : kBoolVarFalse
.v_bool = mpack_unpack_boolean(node->tok) ? kBoolVarTrue : kBoolVarFalse
},
};
break;
case MSGPACK_OBJECT_POSITIVE_INTEGER:
if (mobj.via.u64 <= VARNUMBER_MAX) {
*rettv = (typval_T) {
.v_type = VAR_NUMBER,
.v_lock = VAR_UNLOCKED,
.vval = { .v_number = (varnumber_T)mobj.via.u64 },
};
} else {
list_T *const list = tv_list_alloc(4);
tv_list_ref(list);
create_special_dict(rettv, kMPInteger, ((typval_T) {
.v_type = VAR_LIST,
.v_lock = VAR_UNLOCKED,
.vval = { .v_list = list },
}));
uint64_t n = mobj.via.u64;
tv_list_append_number(list, 1);
tv_list_append_number(list, (varnumber_T)((n >> 62) & 0x3));
tv_list_append_number(list, (varnumber_T)((n >> 31) & 0x7FFFFFFF));
tv_list_append_number(list, (varnumber_T)(n & 0x7FFFFFFF));
}
break;
case MSGPACK_OBJECT_NEGATIVE_INTEGER:
if (mobj.via.i64 >= VARNUMBER_MIN) {
*rettv = (typval_T) {
.v_type = VAR_NUMBER,
.v_lock = VAR_UNLOCKED,
.vval = { .v_number = (varnumber_T)mobj.via.i64 },
};
} else {
list_T *const list = tv_list_alloc(4);
tv_list_ref(list);
create_special_dict(rettv, kMPInteger, ((typval_T) {
.v_type = VAR_LIST,
.v_lock = VAR_UNLOCKED,
.vval = { .v_list = list },
}));
uint64_t n = -((uint64_t)mobj.via.i64);
tv_list_append_number(list, -1);
tv_list_append_number(list, (varnumber_T)((n >> 62) & 0x3));
tv_list_append_number(list, (varnumber_T)((n >> 31) & 0x7FFFFFFF));
tv_list_append_number(list, (varnumber_T)(n & 0x7FFFFFFF));
}
break;
case MSGPACK_OBJECT_FLOAT32:
case MSGPACK_OBJECT_FLOAT64:
*rettv = (typval_T) {
.v_type = VAR_FLOAT,
case MPACK_TOKEN_SINT: {
*result = (typval_T) {
.v_type = VAR_NUMBER,
.v_lock = VAR_UNLOCKED,
.vval = { .v_float = mobj.via.f64 },
.vval = { .v_number = mpack_unpack_sint(node->tok) },
};
break;
case MSGPACK_OBJECT_STR:
case MSGPACK_OBJECT_BIN:
*rettv = decode_string(mobj.via.bin.ptr, mobj.via.bin.size, false, false);
}
case MPACK_TOKEN_UINT:
positive_integer_to_special_typval(result, mpack_unpack_uint(node->tok));
break;
case MSGPACK_OBJECT_ARRAY: {
list_T *const list = tv_list_alloc((ptrdiff_t)mobj.via.array.size);
case MPACK_TOKEN_FLOAT:
*result = (typval_T) {
.v_type = VAR_FLOAT,
.v_lock = VAR_UNLOCKED,
.vval = { .v_float = mpack_unpack_float(node->tok) },
};
break;
case MPACK_TOKEN_BIN:
case MPACK_TOKEN_STR:
case MPACK_TOKEN_EXT:
// actually converted in typval_parse_exit after the data chunks
node->data[1].p = xmallocz(node->tok.length);
break;
case MPACK_TOKEN_CHUNK: {
char *data = parent->data[1].p;
memcpy(data + parent->pos,
node->tok.data.chunk_ptr, node->tok.length);
break;
}
case MPACK_TOKEN_ARRAY: {
list_T *const list = tv_list_alloc((ptrdiff_t)node->tok.length);
tv_list_ref(list);
*rettv = (typval_T) {
*result = (typval_T) {
.v_type = VAR_LIST,
.v_lock = VAR_UNLOCKED,
.vval = { .v_list = list },
};
for (size_t i = 0; i < mobj.via.array.size; i++) {
// Not populated yet, need to create list item to push.
tv_list_append_owned_tv(list, (typval_T) { .v_type = VAR_UNKNOWN });
if (msgpack_to_vim(mobj.via.array.ptr[i],
TV_LIST_ITEM_TV(tv_list_last(list)))
== FAIL) {
return FAIL;
}
}
node->data[1].p = list;
break;
}
case MSGPACK_OBJECT_MAP: {
for (size_t i = 0; i < mobj.via.map.size; i++) {
if ((mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR
&& mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_BIN)
|| mobj.via.map.ptr[i].key.via.str.size == 0
|| memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL,
mobj.via.map.ptr[i].key.via.str.size) != NULL) {
case MPACK_TOKEN_MAP:
// we don't know if this will be safe to convert to a typval dict yet
node->data[1].p = xmallocz(node->tok.length * 2 * sizeof(typval_T));
break;
}
}
/// free node which was entered but never exited, due to a nested error
///
/// Don't bother with typvals as these will be GC:d eventually
void typval_parser_error_free(mpack_parser_t *parser)
{
for (uint32_t i = 0; i < parser->size; i++) {
mpack_node_t *node = &parser->items[i];
switch (node->tok.type) {
case MPACK_TOKEN_BIN:
case MPACK_TOKEN_STR:
case MPACK_TOKEN_EXT:
case MPACK_TOKEN_MAP:
XFREE_CLEAR(node->data[1].p);
break;
default:
break;
}
}
}
static void typval_parse_exit(mpack_parser_t *parser, mpack_node_t *node)
{
typval_T *result = node->data[0].p;
switch (node->tok.type) {
case MPACK_TOKEN_BIN:
case MPACK_TOKEN_STR:
*result = decode_string(node->data[1].p, node->tok.length, false, true);
node->data[1].p = NULL;
break;
case MPACK_TOKEN_EXT: {
list_T *const list = tv_list_alloc(2);
tv_list_ref(list);
tv_list_append_number(list, node->tok.data.ext_type);
list_T *const ext_val_list = tv_list_alloc(kListLenMayKnow);
tv_list_append_list(list, ext_val_list);
create_special_dict(result, kMPExt, ((typval_T) { .v_type = VAR_LIST,
.v_lock = VAR_UNLOCKED,
.vval = { .v_list = list } }));
// TODO(bfredl): why not use BLOB?
encode_list_write((void *)ext_val_list, node->data[1].p, node->tok.length);
XFREE_CLEAR(node->data[1].p);
}
break;
case MPACK_TOKEN_MAP: {
typval_T(*items)[2] = node->data[1].p;
for (size_t i = 0; i < node->tok.length; i++) {
typval_T *key = &items[i][0];
if (key->v_type != VAR_STRING
|| key->vval.v_string == NULL
|| key->vval.v_string[0] == NUL) {
goto msgpack_to_vim_generic_map;
}
}
dict_T *const dict = tv_dict_alloc();
dict->dv_refcount++;
*rettv = (typval_T) {
*result = (typval_T) {
.v_type = VAR_DICT,
.v_lock = VAR_UNLOCKED,
.vval = { .v_dict = dict },
};
for (size_t i = 0; i < mobj.via.map.size; i++) {
dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key)
+ mobj.via.map.ptr[i].key.via.str.size);
memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr,
mobj.via.map.ptr[i].key.via.str.size);
for (size_t i = 0; i < node->tok.length; i++) {
char *key = items[i][0].vval.v_string;
size_t keylen = strlen(key);
dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key) + keylen);
memcpy(&di->di_key[0], key, keylen);
di->di_tv.v_type = VAR_UNKNOWN;
if (tv_dict_add(dict, di) == FAIL) {
// Duplicate key: fallback to generic map
tv_clear(rettv);
TV_DICT_ITER(dict, d, {
d->di_tv.v_type = VAR_SPECIAL; // don't free values in tv_clear(), they will be reused
d->di_tv.vval.v_special = kSpecialVarNull;
});
tv_clear(result);
xfree(di);
goto msgpack_to_vim_generic_map;
}
if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) {
return FAIL;
}
di->di_tv = items[i][1];
}
for (size_t i = 0; i < node->tok.length; i++) {
xfree(items[i][0].vval.v_string);
}
XFREE_CLEAR(node->data[1].p);
break;
msgpack_to_vim_generic_map: {}
list_T *const list = decode_create_map_special_dict(rettv, (ptrdiff_t)mobj.via.map.size);
for (size_t i = 0; i < mobj.via.map.size; i++) {
list_T *const list = decode_create_map_special_dict(result, node->tok.length);
for (size_t i = 0; i < node->tok.length; i++) {
list_T *const kv_pair = tv_list_alloc(2);
tv_list_append_list(list, kv_pair);
typval_T key_tv = { .v_type = VAR_UNKNOWN };
if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_tv) == FAIL) {
tv_clear(&key_tv);
return FAIL;
}
tv_list_append_owned_tv(kv_pair, key_tv);
tv_list_append_owned_tv(kv_pair, items[i][0]);
tv_list_append_owned_tv(kv_pair, items[i][1]);
}
XFREE_CLEAR(node->data[1].p);
break;
}
typval_T val_tv = { .v_type = VAR_UNKNOWN };
if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_tv) == FAIL) {
tv_clear(&val_tv);
return FAIL;
}
tv_list_append_owned_tv(kv_pair, val_tv);
}
default:
// other kinds are handled completely in typval_parse_enter,
break;
}
case MSGPACK_OBJECT_EXT: {
list_T *const list = tv_list_alloc(2);
tv_list_ref(list);
tv_list_append_number(list, mobj.via.ext.type);
list_T *const ext_val_list = tv_list_alloc(kListLenMayKnow);
tv_list_append_list(list, ext_val_list);
create_special_dict(rettv, kMPExt, ((typval_T) { .v_type = VAR_LIST,
.v_lock = VAR_UNLOCKED,
.vval = { .v_list = list } }));
if (encode_list_write((void *)ext_val_list, mobj.via.ext.ptr,
mobj.via.ext.size) == -1) {
return FAIL;
}
break;
}
}
return OK;
}
int mpack_parse_typval(mpack_parser_t *parser, const char **data, size_t *size)
{
return mpack_parse(parser, data, size, typval_parse_enter, typval_parse_exit);
}
int unpack_typval(const char **data, size_t *size, typval_T *ret)
{
ret->v_type = VAR_UNKNOWN;
mpack_parser_t parser;
mpack_parser_init(&parser, 0);
parser.data.p = ret;
int status = mpack_parse_typval(&parser, data, size);
if (status != MPACK_OK) {
typval_parser_error_free(&parser);
tv_clear(ret);
}
return status;
}

2
src/nvim/eval/decode.h
View File

@ -1,8 +1,8 @@
#pragma once
#include <msgpack.h> // IWYU pragma: keep
#include <stddef.h> // IWYU pragma: keep
#include "mpack/object.h"
#include "nvim/eval/typval_defs.h" // IWYU pragma: keep
#include "nvim/types_defs.h" // IWYU pragma: keep

5
src/nvim/eval/encode.c
View File

@ -55,11 +55,11 @@ int encode_blob_write(void *const data, const char *const buf, const size_t len)
}
/// Msgpack callback for writing to readfile()-style list
int encode_list_write(void *const data, const char *const buf, const size_t len)
void encode_list_write(void *const data, const char *const buf, const size_t len)
FUNC_ATTR_NONNULL_ARG(1)
{
if (len == 0) {
return 0;
return;
}
list_T *const list = (list_T *)data;
const char *const end = buf + len;
@ -97,7 +97,6 @@ int encode_list_write(void *const data, const char *const buf, const size_t len)
if (line_end == end) {
tv_list_append_allocated_string(list, NULL);
}
return 0;
}
/// Abort conversion to string after a recursion error.

1
src/nvim/eval/encode.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <msgpack/pack.h>
#include <string.h>
#include "nvim/eval/typval_defs.h"

130
src/nvim/eval/funcs.c
View File

@ -4,9 +4,6 @@
#include <inttypes.h>
#include <limits.h>
#include <math.h>
#include <msgpack/object.h>
#include <msgpack/pack.h>
#include <msgpack/unpack.h>
#include <signal.h>
#include <stdarg.h>
#include <stddef.h>
@ -18,6 +15,7 @@
#include <uv.h>
#include "auto/config.h"
#include "mpack/object.h"
#include "nvim/api/private/converter.h"
#include "nvim/api/private/defs.h"
#include "nvim/api/private/dispatch.h"
@ -5723,36 +5721,24 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
}
}
static int msgpackparse_convert_item(const msgpack_object data, const msgpack_unpack_return result,
list_T *const ret_list, const bool fail_if_incomplete)
FUNC_ATTR_NONNULL_ALL
static void emsg_mpack_error(int status)
{
switch (result) {
case MSGPACK_UNPACK_PARSE_ERROR:
switch (status) {
case MPACK_ERROR:
semsg(_(e_invarg2), "Failed to parse msgpack string");
return FAIL;
case MSGPACK_UNPACK_NOMEM_ERROR:
emsg(_(e_outofmem));
return FAIL;
case MSGPACK_UNPACK_CONTINUE:
if (fail_if_incomplete) {
semsg(_(e_invarg2), "Incomplete msgpack string");
return FAIL;
}
return NOTDONE;
case MSGPACK_UNPACK_SUCCESS: {
typval_T tv = { .v_type = VAR_UNKNOWN };
if (msgpack_to_vim(data, &tv) == FAIL) {
semsg(_(e_invarg2), "Failed to convert msgpack string");
return FAIL;
}
tv_list_append_owned_tv(ret_list, tv);
return OK;
break;
case MPACK_EOF:
semsg(_(e_invarg2), "Incomplete msgpack string");
break;
case MPACK_NOMEM:
semsg(_(e_invarg2), "object was too deep to unpack");
break;
default:
break;
}
case MSGPACK_UNPACK_EXTRA_BYTES:
abort();
}
UNREACHABLE;
}
static void msgpackparse_unpack_list(const list_T *const list, list_T *const ret_list)
@ -5766,48 +5752,57 @@ static void msgpackparse_unpack_list(const list_T *const list, list_T *const ret
return;
}
ListReaderState lrstate = encode_init_lrstate(list);
msgpack_unpacker *const unpacker = msgpack_unpacker_new(IOSIZE);
if (unpacker == NULL) {
emsg(_(e_outofmem));
return;
}
msgpack_unpacked unpacked;
msgpack_unpacked_init(&unpacked);
char *buf = alloc_block();
size_t buf_size = 0;
typval_T cur_item = { .v_type = VAR_UNKNOWN };
mpack_parser_t parser;
mpack_parser_init(&parser, 0);
parser.data.p = &cur_item;
int status = MPACK_OK;
while (true) {
if (!msgpack_unpacker_reserve_buffer(unpacker, IOSIZE)) {
emsg(_(e_outofmem));
goto end;
}
size_t read_bytes;
const int rlret = encode_read_from_list(&lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE,
const int rlret = encode_read_from_list(&lrstate, buf + buf_size, ARENA_BLOCK_SIZE - buf_size,
&read_bytes);
if (rlret == FAIL) {
semsg(_(e_invarg2), "List item is not a string");
goto end;
}
msgpack_unpacker_buffer_consumed(unpacker, read_bytes);
if (read_bytes == 0) {
break;
}
while (unpacker->off < unpacker->used) {
const msgpack_unpack_return result
= msgpack_unpacker_next(unpacker, &unpacked);
const int conv_result = msgpackparse_convert_item(unpacked.data, result,
ret_list, rlret == OK);
if (conv_result == NOTDONE) {
buf_size += read_bytes;
const char *ptr = buf;
while (buf_size) {
status = mpack_parse_typval(&parser, &ptr, &buf_size);
if (status == MPACK_OK) {
tv_list_append_owned_tv(ret_list, cur_item);
cur_item.v_type = VAR_UNKNOWN;
} else {
break;
} else if (conv_result == FAIL) {
goto end;
}
}
if (rlret == OK) {
break;
}
if (status == MPACK_EOF) {
// move remaining data to front of buffer
if (buf_size && ptr > buf) {
memmove(buf, ptr, buf_size);
}
} else if (status != MPACK_OK) {
break;
}
}
if (status != MPACK_OK) {
typval_parser_error_free(&parser);
emsg_mpack_error(status);
}
end:
msgpack_unpacker_free(unpacker);
msgpack_unpacked_destroy(&unpacked);
free_block(buf);
}
static void msgpackparse_unpack_blob(const blob_T *const blob, list_T *const ret_list)
@ -5817,18 +5812,19 @@ static void msgpackparse_unpack_blob(const blob_T *const blob, list_T *const ret
if (len == 0) {
return;
}
msgpack_unpacked unpacked;
msgpack_unpacked_init(&unpacked);
for (size_t offset = 0; offset < (size_t)len;) {
const msgpack_unpack_return result
= msgpack_unpack_next(&unpacked, blob->bv_ga.ga_data, (size_t)len, &offset);
if (msgpackparse_convert_item(unpacked.data, result, ret_list, true)
!= OK) {
break;
}
}
msgpack_unpacked_destroy(&unpacked);
const char *data = blob->bv_ga.ga_data;
size_t remaining = (size_t)len;
while (remaining) {
typval_T tv;
int status = unpack_typval(&data, &remaining, &tv);
if (status != MPACK_OK) {
emsg_mpack_error(status);
return;
}
tv_list_append_owned_tv(ret_list, tv);
}
}
/// "msgpackparse" function

5
src/nvim/eval/typval.c
View File

@ -485,13 +485,14 @@ void tv_list_append_tv(list_T *const l, typval_T *const tv)
/// Like tv_list_append_tv(), but tv is moved to a list
///
/// This means that it is no longer valid to use contents of the typval_T after
/// function exits.
void tv_list_append_owned_tv(list_T *const l, typval_T tv)
/// function exits. A pointer is returned to the allocated typval which can be used
typval_T *tv_list_append_owned_tv(list_T *const l, typval_T tv)
FUNC_ATTR_NONNULL_ALL
{
listitem_T *const li = tv_list_item_alloc();
*TV_LIST_ITEM_TV(li) = tv;
tv_list_append(l, li);
return TV_LIST_ITEM_TV(li);
}
/// Append a list to a list as one item

6
src/nvim/ex_cmds.c
View File

@ -3068,8 +3068,8 @@ void sub_get_replacement(SubReplacementString *const ret_sub)
void sub_set_replacement(SubReplacementString sub)
{
xfree(old_sub.sub);
if (sub.additional_elements != old_sub.additional_elements) {
tv_list_unref(old_sub.additional_elements);
if (sub.additional_data != old_sub.additional_data) {
xfree(old_sub.additional_data);
}
old_sub = sub;
}
@ -3395,7 +3395,7 @@ static int do_sub(exarg_T *eap, const proftime_T timeout, const int cmdpreview_n
sub_set_replacement((SubReplacementString) {
.sub = xstrdup(sub),
.timestamp = os_time(),
.additional_elements = NULL,
.additional_data = NULL,
});
}
} else if (!eap->skip) { // use previous pattern and substitution

2
src/nvim/ex_cmds_defs.h
View File

@ -233,5 +233,5 @@ typedef struct {
typedef struct {
char *sub; ///< Previous replacement string.
Timestamp timestamp; ///< Time when it was last set.
list_T *additional_elements; ///< Additional data left from ShaDa file.
AdditionalData *additional_data; ///< Additional data left from ShaDa file.
} SubReplacementString;

6
src/nvim/generators/c_grammar.lua
View File

@ -91,7 +91,9 @@ local c_proto = Ct(
* (P(';') + (P('{') * nl + (impl_line ^ 0) * P('}')))
)
local c_field = Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * P(';') * fill)
local dict_key = P('DictKey(') * Cg(rep1(any - P(')')), 'dict_key') * P(')')
local keyset_field =
Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * (dict_key ^ -1) * fill * P(';') * fill)
local c_keyset = Ct(
P('typedef')
* ws
@ -99,7 +101,7 @@ local c_keyset = Ct(
* fill
* P('{')
* fill
* Cg(Ct(c_field ^ 1), 'fields')
* Cg(Ct(keyset_field ^ 1), 'fields')
* P('}')
* fill
* P('Dict')

27
src/nvim/generators/gen_api_dispatch.lua
View File

@ -72,14 +72,19 @@ local keysets = {}
local function add_keyset(val)
local keys = {}
local types = {}
local c_names = {}
local is_set_name = 'is_set__' .. val.keyset_name .. '_'
local has_optional = false
for i, field in ipairs(val.fields) do
local dict_key = field.dict_key or field.name
if field.type ~= 'Object' then
types[field.name] = field.type
types[dict_key] = field.type
end
if field.name ~= is_set_name and field.type ~= 'OptionalKeys' then
table.insert(keys, field.name)
table.insert(keys, dict_key)
if dict_key ~= field.name then
c_names[dict_key] = field.name
end
else
if i > 1 then
error("'is_set__{type}_' must be first if present")
@ -94,6 +99,7 @@ local function add_keyset(val)
table.insert(keysets, {
name = val.keyset_name,
keys = keys,
c_names = c_names,
types = types,
has_optional = has_optional,
})
@ -332,19 +338,6 @@ output:write([[
keysets_defs:write('// IWYU pragma: private, include "nvim/api/private/dispatch.h"\n\n')
for _, k in ipairs(keysets) do
local c_name = {}
for i = 1, #k.keys do
-- some keys, like "register" are c keywords and get
-- escaped with a trailing _ in the struct.
if vim.endswith(k.keys[i], '_') then
local orig = k.keys[i]
k.keys[i] = string.sub(k.keys[i], 1, #k.keys[i] - 1)
c_name[k.keys[i]] = orig
k.types[k.keys[i]] = k.types[orig]
end
end
local neworder, hashfun = hashy.hashy_hash(k.name, k.keys, function(idx)
return k.name .. '_table[' .. idx .. '].str'
end)
@ -354,6 +347,8 @@ for _, k in ipairs(keysets) do
local function typename(type)
if type == 'HLGroupID' then
return 'kObjectTypeInteger'
elseif type == 'StringArray' then
return 'kUnpackTypeStringArray'
elseif type ~= nil then
return 'kObjectType' .. type
else
@ -374,7 +369,7 @@ for _, k in ipairs(keysets) do
.. '", offsetof(KeyDict_'
.. k.name
.. ', '
.. (c_name[key] or key)
.. (k.c_names[key] or key)
.. '), '
.. typename(k.types[key])
.. ', '

22
src/nvim/generators/hashy.lua
View File

@ -73,11 +73,15 @@ function M.switcher(put, tab, maxlen, worst_buck_size)
vim.list_extend(neworder, buck)
local endidx = #neworder
put(" case '" .. c .. "': ")
put('low = ' .. startidx .. '; ')
if bucky then
put('high = ' .. endidx .. '; ')
if len == 1 then
put('return ' .. startidx .. ';\n')
else
put('low = ' .. startidx .. '; ')
if bucky then
put('high = ' .. endidx .. '; ')
end
put 'break;\n'
end
put 'break;\n'
end
put ' default: break;\n'
put ' }\n '
@ -105,13 +109,19 @@ function M.hashy_hash(name, strings, access)
end
local len_pos_buckets, maxlen, worst_buck_size = M.build_pos_hash(strings)
put('int ' .. name .. '_hash(const char *str, size_t len)\n{\n')
if worst_buck_size > 1 then
if maxlen == 1 then
put('\n') -- nothing
elseif worst_buck_size > 1 then
put(' int low = 0, high = 0;\n')
else
put(' int low = -1;\n')
end
local neworder = M.switcher(put, len_pos_buckets, maxlen, worst_buck_size)
if worst_buck_size > 1 then
if maxlen == 1 then
put([[
return -1;
]])
elseif worst_buck_size > 1 then
put([[
for (int i = low; i < high; i++) {
if (!memcmp(str, ]] .. access('i') .. [[, len)) {

1
src/nvim/main.c
View File

@ -6,7 +6,6 @@
#endif
#include <assert.h>
#include <limits.h>
#include <msgpack/pack.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>

2
src/nvim/mark.c
View File

@ -71,7 +71,7 @@ int setmark(int c)
/// Free fmark_T item
void free_fmark(fmark_T fm)
{
tv_dict_unref(fm.additional_data);
xfree(fm.additional_data);
}
/// Free xfmark_T item

8
src/nvim/mark_defs.h
View File

@ -1,7 +1,11 @@
#pragma once
#include "nvim/eval/typval_defs.h"
#include <stdbool.h>
#include "nvim/func_attr.h"
#include "nvim/os/time_defs.h"
#include "nvim/pos_defs.h"
#include "nvim/types_defs.h"
#ifdef INCLUDE_GENERATED_DECLARATIONS
# include "mark_defs.h.inline.generated.h"
@ -75,7 +79,7 @@ typedef struct {
int fnum; ///< File number.
Timestamp timestamp; ///< Time when this mark was last set.
fmarkv_T view; ///< View the mark was created on
dict_T *additional_data; ///< Additional data from ShaDa file.
AdditionalData *additional_data; ///< Additional data from ShaDa file.
} fmark_T;
#define INIT_FMARK { { 0, 0, 0 }, 0, 0, INIT_FMARKV, NULL }

3
src/nvim/msgpack_rpc/channel.c
View File

@ -1,8 +1,5 @@
#include <assert.h>
#include <inttypes.h>
#include <msgpack/object.h>
#include <msgpack/sbuffer.h>
#include <msgpack/unpack.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

1
src/nvim/msgpack_rpc/channel_defs.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <msgpack.h>
#include <stdbool.h>
#include <uv.h>

196
src/nvim/msgpack_rpc/unpacker.c
View File

@ -11,6 +11,7 @@
#include "nvim/memory.h"
#include "nvim/msgpack_rpc/channel_defs.h"
#include "nvim/msgpack_rpc/unpacker.h"
#include "nvim/strings.h"
#include "nvim/ui_client.h"
#ifdef INCLUDE_GENERATED_DECLARATIONS
@ -206,6 +207,7 @@ bool unpacker_parse_header(Unpacker *p)
assert(!ERROR_SET(&p->unpack_error));
// TODO(bfredl): eliminate p->reader, we can use mpack_rtoken directly
#define NEXT(tok) \
result = mpack_read(&p->reader, &data, &size, &tok); \
if (result) { goto error; }
@ -522,3 +524,197 @@ bool unpacker_parse_redraw(Unpacker *p)
abort();
}
}
/// require complete string. safe to use e.g. in shada as we have loaded a complete shada item into
/// a linear buffer.
///
/// data and size are preserved in cause of failure
///
/// @return "data" is NULL exact when failure (non-null data and size=0 for
/// valid empty string)
String unpack_string(const char **data, size_t *size)
{
const char *data2 = *data;
size_t size2 = *size;
mpack_token_t tok;
// TODO(bfredl): this code is hot a f, specialize!
int result = mpack_rtoken(&data2, &size2, &tok);
if (result || (tok.type != MPACK_TOKEN_STR && tok.type != MPACK_TOKEN_BIN)) {
return (String)STRING_INIT;
}
if (*size < tok.length) {
// result = MPACK_EOF;
return (String)STRING_INIT;
}
(*data) = data2 + tok.length;
(*size) = size2 - tok.length;
return cbuf_as_string((char *)data2, tok.length);
}
/// @return -1 if not an array or EOF. otherwise size of valid array
ssize_t unpack_array(const char **data, size_t *size)
{
// TODO(bfredl): this code is hot, specialize!
mpack_token_t tok;
int result = mpack_rtoken(data, size, &tok);
if (result || tok.type != MPACK_TOKEN_ARRAY) {
return -1;
}
return tok.length;
}
/// does not keep "data" untouched on failure
bool unpack_integer(const char **data, size_t *size, Integer *res)
{
mpack_token_t tok;
int result = mpack_rtoken(data, size, &tok);
if (result) {
return false;
}
return unpack_uint_or_sint(tok, res);
}
bool unpack_uint_or_sint(mpack_token_t tok, Integer *res)
{
if (tok.type == MPACK_TOKEN_UINT) {
*res = (Integer)mpack_unpack_uint(tok);
return true;
} else if (tok.type == MPACK_TOKEN_SINT) {
*res = (Integer)mpack_unpack_sint(tok);
return true;
}
return false;
}
static void parse_nop(mpack_parser_t *parser, mpack_node_t *node)
{
}
int unpack_skip(const char **data, size_t *size)
{
mpack_parser_t parser;
mpack_parser_init(&parser, 0);
return mpack_parse(&parser, data, size, parse_nop, parse_nop);
}
void push_additional_data(AdditionalDataBuilder *ad, const char *data, size_t size)
{
if (kv_size(*ad) == 0) {
AdditionalData init = { 0 };
kv_concat_len(*ad, &init, sizeof(init));
}
AdditionalData *a = (AdditionalData *)ad->items;
a->nitems++;
a->nbytes += (uint32_t)size;
kv_concat_len(*ad, data, size);
}
// currently only used for shada, so not re-entrant like unpacker_parse_redraw
bool unpack_keydict(void *retval, FieldHashfn hashy, AdditionalDataBuilder *ad, const char **data,
size_t *restrict size, char **error)
{
OptKeySet *ks = (OptKeySet *)retval;
mpack_token_t tok;
int result = mpack_rtoken(data, size, &tok);
if (result || tok.type != MPACK_TOKEN_MAP) {
*error = xstrdup("is not a dictionary");
return false;
}
size_t map_size = tok.length;
for (size_t i = 0; i < map_size; i++) {
const char *item_start = *data;
// TODO(bfredl): we could specialize a hot path for FIXSTR here
String key = unpack_string(data, size);
if (!key.data) {
*error = arena_printf(NULL, "has key value which is not a string").data;
return false;
} else if (key.size == 0) {
*error = arena_printf(NULL, "has empty key").data;
return false;
}
KeySetLink *field = hashy(key.data, key.size);
if (!field) {
int status = unpack_skip(data, size);
if (status) {
return false;
}
if (ad) {
push_additional_data(ad, item_start, (size_t)(*data - item_start));
}
continue;
}
assert(field->opt_index >= 0);
uint64_t flag = (1ULL << field->opt_index);
if (ks->is_set_ & flag) {
*error = xstrdup("duplicate key");
return false;
}
ks->is_set_ |= flag;
char *mem = ((char *)retval + field->ptr_off);
switch (field->type) {
case kObjectTypeBoolean:
if (*size == 0 || (**data & 0xfe) != 0xc2) {
*error = arena_printf(NULL, "has %.*s key value which is not a boolean", (int)key.size,
key.data).data;
return false;
}
*(Boolean *)mem = **data & 0x01;
(*data)++; (*size)--;
break;
case kObjectTypeInteger:
if (!unpack_integer(data, size, (Integer *)mem)) {
*error = arena_printf(NULL, "has %.*s key value which is not an integer", (int)key.size,
key.data).data;
return false;
}
break;
case kObjectTypeString: {
String val = unpack_string(data, size);
if (!val.data) {
*error = arena_printf(NULL, "has %.*s key value which is not a binary", (int)key.size,
key.data).data;
return false;
}
*(String *)mem = val;
break;
}
case kUnpackTypeStringArray: {
ssize_t len = unpack_array(data, size);
if (len < 0) {
*error = arena_printf(NULL, "has %.*s key with non-array value", (int)key.size,
key.data).data;
return false;
}
StringArray *a = (StringArray *)mem;
kv_ensure_space(*a, (size_t)len);
for (size_t j = 0; j < (size_t)len; j++) {
String item = unpack_string(data, size);
if (!item.data) {
*error = arena_printf(NULL, "has %.*s array with non-binary value", (int)key.size,
key.data).data;
return false;
}
kv_push(*a, item);
}
break;
}
default:
abort(); // not supported
}
}
return true;
}

2
src/nvim/msgpack_rpc/unpacker.h
View File

@ -41,6 +41,8 @@ struct Unpacker {
bool has_grid_line_event;
};
typedef kvec_t(char) AdditionalDataBuilder;
// unrecovareble error. unpack_error should be set!
#define unpacker_closed(p) ((p)->state < 0)

16
src/nvim/ops.c
View File

@ -930,16 +930,6 @@ int do_record(int c)
return retval;
}
static void set_yreg_additional_data(yankreg_T *reg, dict_T *additional_data)
FUNC_ATTR_NONNULL_ARG(1)
{
if (reg->additional_data == additional_data) {
return;
}
tv_dict_unref(reg->additional_data);
reg->additional_data = additional_data;
}
/// Stuff string "p" into yank register "regname" as a single line (append if
/// uppercase). "p" must have been allocated.
///
@ -969,7 +959,7 @@ static int stuff_yank(int regname, char *p)
*pp = lp;
} else {
free_register(reg);
set_yreg_additional_data(reg, NULL);
reg->additional_data = NULL;
reg->y_array = xmalloc(sizeof(char *));
reg->y_array[0] = p;
reg->y_size = 1;
@ -2507,7 +2497,7 @@ void clear_registers(void)
void free_register(yankreg_T *reg)
FUNC_ATTR_NONNULL_ALL
{
set_yreg_additional_data(reg, NULL);
XFREE_CLEAR(reg->additional_data);
if (reg->y_array == NULL) {
return;
}
@ -5144,7 +5134,7 @@ static void str_to_reg(yankreg_T *y_ptr, MotionType yank_type, const char *str,
}
y_ptr->y_type = yank_type;
y_ptr->y_size = lnum;
set_yreg_additional_data(y_ptr, NULL);
XFREE_CLEAR(y_ptr->additional_data);
y_ptr->timestamp = os_time();
if (yank_type == kMTBlockWise) {
y_ptr->y_width = (blocklen == -1 ? (colnr_T)maxlen - 1 : blocklen);

2
src/nvim/ops.h
View File

@ -110,7 +110,7 @@ typedef struct {
MotionType y_type; ///< Register type
colnr_T y_width; ///< Register width (only valid for y_type == kBlockWise).
Timestamp timestamp; ///< Time when register was last modified.
dict_T *additional_data; ///< Additional data from ShaDa file.
AdditionalData *additional_data; ///< Additional data from ShaDa file.
} yankreg_T;
/// Modes for get_yank_register()

16
src/nvim/os/fileio.c
View File

@ -309,6 +309,22 @@ ptrdiff_t file_read(FileDescriptor *const fp, char *const ret_buf, const size_t
return (ptrdiff_t)(size - read_remaining);
}
/// try to read already buffered data in place
///
/// @return NULL if enough data is not available
/// valid pointer to chunk of "size". pointer becomes invalid in the next "file_read" call!
char *file_try_read_buffered(FileDescriptor *const fp, const size_t size)
FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
{
if ((size_t)(fp->write_pos - fp->read_pos) >= size) {
char *ret = fp->read_pos;
fp->read_pos += size;
fp->bytes_read += size;
return ret;
}
return NULL;
}
/// Write to a file
///
/// @param[in] fd File descriptor to write to.

2
src/nvim/search.c
View File

@ -290,7 +290,7 @@ void restore_search_patterns(void)
static inline void free_spat(SearchPattern *const spat)
{
xfree(spat->pat);
tv_dict_unref(spat->additional_data);
xfree(spat->additional_data);
}
#if defined(EXITFREE)

2
src/nvim/search.h
View File

@ -89,7 +89,7 @@ typedef struct {
bool no_scs; ///< No smartcase for this pattern.
Timestamp timestamp; ///< Time of the last change.
SearchOffset off; ///< Pattern offset.
dict_T *additional_data; ///< Additional data from ShaDa file.
AdditionalData *additional_data; ///< Additional data from ShaDa file.
} SearchPattern;
/// Optional extra arguments for searchit().

1079
src/nvim/shada.c
View File

File diff suppressed because it is too large Load Diff

2
src/nvim/shada.h
View File

@ -1,7 +1,5 @@
#pragma once
#include <msgpack.h> // IWYU pragma: keep
#include "nvim/api/private/defs.h"
/// Flags for shada_read_file and children

6
src/nvim/types_defs.h
View File

@ -56,3 +56,9 @@ typedef struct regprog regprog_T;
typedef struct syn_state synstate_T;
typedef struct terminal Terminal;
typedef struct window_S win_T;
typedef struct {
uint32_t nitems;
uint32_t nbytes;
char data[];
} AdditionalData;

2
src/nvim/ui_client.c
View File

@ -174,7 +174,7 @@ static HlAttrs ui_client_dict2hlattrs(Dictionary d, bool rgb)
{
Error err = ERROR_INIT;
Dict(highlight) dict = KEYDICT_INIT;
if (!api_dict_to_keydict(&dict, KeyDict_highlight_get_field, d, &err)) {
if (!api_dict_to_keydict(&dict, DictHash(highlight), d, &err)) {
// TODO(bfredl): log "err"
return HLATTRS_INIT;
}

3
src/nvim/undo.c
View File

@ -343,8 +343,7 @@ static OptInt get_undolevel(buf_T *buf)
static inline void zero_fmark_additional_data(fmark_T *fmarks)
{
for (size_t i = 0; i < NMARKS; i++) {
tv_dict_unref(fmarks[i].additional_data);
fmarks[i].additional_data = NULL;
XFREE_CLEAR(fmarks[i].additional_data);
}
}

95
test/functional/shada/errors_spec.lua
View File

@ -58,7 +58,7 @@ describe('ShaDa error handling', function()
it('fails on search pattern item with zero length', function()
wshada('\002\000\000')
eq(
'Vim(rshada):E576: Failed to parse ShaDa file: incomplete msgpack string at position 3',
'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 is not a dictionary',
exc_exec(sdrcmd())
)
end)
@ -89,18 +89,10 @@ describe('ShaDa error handling', function()
it('fails on search pattern item with invalid byte', function()
-- 195 (== 0xC1) cannot start any valid messagepack entry (the only byte
-- that cannot do this). Specifically unpack_template.h contains
--
-- //case 0xc1: // string
-- // again_terminal_trail(NEXT_CS(p), p+1);
--
-- (literally: commented out code) which means that in place of this code
-- `goto _failed` is used from default: case. I do not know any other way to
-- get MSGPACK_UNPACK_PARSE_ERROR and not MSGPACK_UNPACK_CONTINUE or
-- MSGPACK_UNPACK_EXTRA_BYTES.
-- that cannot do this)
wshada('\002\000\001\193')
eq(
'Vim(rshada):E576: Failed to parse ShaDa file due to a msgpack parser error at position 3',
'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 is not a dictionary',
exc_exec(sdrcmd())
)
end)
@ -108,7 +100,7 @@ describe('ShaDa error handling', function()
it('fails on search pattern item with incomplete map', function()
wshada('\002\000\001\129')
eq(
'Vim(rshada):E576: Failed to parse ShaDa file: incomplete msgpack string at position 3',
'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has key value which is not a string',
exc_exec(sdrcmd())
)
end)
@ -124,7 +116,7 @@ describe('ShaDa error handling', function()
it('fails on search pattern with extra bytes', function()
wshada('\002\000\002\128\000')
eq(
'Vim(rshada):E576: Failed to parse ShaDa file: extra bytes in msgpack string at position 3',
'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has no pattern',
exc_exec(sdrcmd())
)
end)
@ -137,15 +129,6 @@ describe('ShaDa error handling', function()
)
end)
-- sp entry is here because it causes an allocation.
it('fails on search pattern item with BIN key', function()
wshada('\002\000\014\131\162sp\196\001a\162sX\192\196\000\000')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has key which is not a string',
exc_exec(sdrcmd())
)
end)
-- sp entry is here because it causes an allocation.
it('fails on search pattern item with empty key', function()
wshada('\002\000\013\131\162sp\196\001a\162sX\192\160\000')
@ -235,22 +218,12 @@ describe('ShaDa error handling', function()
)
end)
it('fails on search pattern item with STR pat key value', function()
wshada('\002\000\011\130\162sX\192\162sp\162sp')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sp key value which is not a binary',
exc_exec(sdrcmd())
)
end)
for _, v in ipairs({
{ name = 'global mark', mpack = '\007' },
{ name = 'jump', mpack = '\008' },
{ name = 'local mark', mpack = '\010' },
{ name = 'change', mpack = '\011' },
}) do
local is_mark_test = ({ ['global mark'] = true, ['local mark'] = true })[v.name]
it('fails on ' .. v.name .. ' item with NIL value', function()
wshada(v.mpack .. '\000\001\192')
eq(
@ -259,15 +232,6 @@ describe('ShaDa error handling', function()
)
end)
-- f entry is here because it causes an allocation.
it('fails on ' .. v.name .. ' item with BIN key', function()
wshada(v.mpack .. '\000\013\131\161f\196\001/\162mX\192\196\000\000')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has key which is not a string',
exc_exec(sdrcmd())
)
end)
-- f entry is here because it causes an allocation.
it('fails on ' .. v.name .. ' item with empty key', function()
wshada(v.mpack .. '\000\012\131\161f\196\001/\162mX\192\160\000')
@ -312,9 +276,7 @@ describe('ShaDa error handling', function()
it('fails on ' .. v.name .. ' item with STR n key value', function()
wshada(v.mpack .. '\000\011\130\162mX\192\161n\163spa')
eq(
is_mark_test
and 'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key value which is not an unsigned integer'
or 'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key which is only valid for local and global mark entries',
'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key value which is not an integer',
exc_exec(sdrcmd())
)
end)
@ -334,14 +296,6 @@ describe('ShaDa error handling', function()
exc_exec(sdrcmd())
)
end)
it('fails on ' .. v.name .. ' item with STR f key value', function()
wshada(v.mpack .. '\000\010\130\162mX\192\161f\162sp')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has f key value which is not a binary',
exc_exec(sdrcmd())
)
end)
end
it('fails on register item with NIL value', function()
@ -352,15 +306,6 @@ describe('ShaDa error handling', function()
)
end)
-- rc entry is here because it causes an allocation
it('fails on register item with BIN key', function()
wshada('\005\000\015\131\162rc\145\196\001a\162rX\192\196\000\000')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has key which is not a string',
exc_exec(sdrcmd())
)
end)
-- rc entry is here because it causes an allocation
it('fails on register item with BIN key', function()
wshada('\005\000\014\131\162rc\145\196\001a\162rX\192\160\000')
@ -373,7 +318,7 @@ describe('ShaDa error handling', function()
it('fails on register item with NIL rt key value', function()
wshada('\005\000\009\130\162rX\192\162rt\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rt key value which is not an unsigned integer',
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rt key value which is not an integer',
exc_exec(sdrcmd())
)
end)
@ -381,7 +326,7 @@ describe('ShaDa error handling', function()
it('fails on register item with NIL rw key value', function()
wshada('\005\000\009\130\162rX\192\162rw\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rw key value which is not an unsigned integer',
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rw key value which is not an integer',
exc_exec(sdrcmd())
)
end)
@ -397,7 +342,7 @@ describe('ShaDa error handling', function()
it('fails on register item with empty rc key value', function()
wshada('\005\000\009\130\162rX\192\162rc\144')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with empty array',
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with missing or empty array',
exc_exec(sdrcmd())
)
end)
@ -413,7 +358,7 @@ describe('ShaDa error handling', function()
it('fails on register item without rc array', function()
wshada('\005\000\009\129\162rX\146\196\001a\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has missing rc array',
'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with missing or empty array',
exc_exec(sdrcmd())
)
end)
@ -421,7 +366,7 @@ describe('ShaDa error handling', function()
it('fails on history item with NIL value', function()
wshada('\004\000\001\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array',
'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -429,7 +374,7 @@ describe('ShaDa error handling', function()
it('fails on history item with empty value', function()
wshada('\004\000\001\144')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 does not have enough elements',
'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -437,7 +382,7 @@ describe('ShaDa error handling', function()
it('fails on history item with single element value', function()
wshada('\004\000\002\145\000')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 does not have enough elements',
'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -485,7 +430,7 @@ describe('ShaDa error handling', function()
it('fails on variable item with NIL value', function()
wshada('\006\000\001\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array',
'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -493,7 +438,7 @@ describe('ShaDa error handling', function()
it('fails on variable item with empty value', function()
wshada('\006\000\001\144')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 does not have enough elements',
'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -501,7 +446,7 @@ describe('ShaDa error handling', function()
it('fails on variable item with single element value', function()
wshada('\006\000\002\145\000')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 does not have enough elements',
'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -525,7 +470,7 @@ describe('ShaDa error handling', function()
it('fails on replacement item with NIL value', function()
wshada('\003\000\001\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array',
'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -533,7 +478,7 @@ describe('ShaDa error handling', function()
it('fails on replacement item with empty value', function()
wshada('\003\000\001\144')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 does not have enough elements',
'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array with enough elements',
exc_exec(sdrcmd())
)
end)
@ -577,7 +522,7 @@ describe('ShaDa error handling', function()
nvim_command('set shada+=%')
wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161l\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: buffer list entry entry at position 0 has l key value which is not an integer',
'Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry that has l key value which is not an integer',
exc_exec(sdrcmd())
)
end)
@ -613,7 +558,7 @@ describe('ShaDa error handling', function()
nvim_command('set shada+=%')
wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161c\192')
eq(
'Vim(rshada):E575: Error while reading ShaDa file: buffer list entry entry at position 0 has c key value which is not an integer',
'Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry that has c key value which is not an integer',
exc_exec(sdrcmd())
)
end)

36
test/unit/eval/encode_spec.lua
View File

@ -21,81 +21,81 @@ describe('encode_list_write()', function()
itp('writes empty string', function()
local l = list()
eq(0, encode_list_write(l, ''))
encode_list_write(l, '')
eq({ [type_key] = list_type }, lst2tbl(l))
end)
itp('writes ASCII string literal with printable characters', function()
local l = list()
eq(0, encode_list_write(l, 'abc'))
encode_list_write(l, 'abc')
eq({ 'abc' }, lst2tbl(l))
end)
itp('writes string starting with NL', function()
local l = list()
eq(0, encode_list_write(l, '\nabc'))
encode_list_write(l, '\nabc')
eq({ null_string, 'abc' }, lst2tbl(l))
end)
itp('writes string starting with NL twice', function()
local l = list()
eq(0, encode_list_write(l, '\nabc'))
encode_list_write(l, '\nabc')
eq({ null_string, 'abc' }, lst2tbl(l))
eq(0, encode_list_write(l, '\nabc'))
encode_list_write(l, '\nabc')
eq({ null_string, 'abc', 'abc' }, lst2tbl(l))
end)
itp('writes string ending with NL', function()
local l = list()
eq(0, encode_list_write(l, 'abc\n'))
encode_list_write(l, 'abc\n')
eq({ 'abc', null_string }, lst2tbl(l))
end)
itp('writes string ending with NL twice', function()
local l = list()
eq(0, encode_list_write(l, 'abc\n'))
encode_list_write(l, 'abc\n')
eq({ 'abc', null_string }, lst2tbl(l))
eq(0, encode_list_write(l, 'abc\n'))
encode_list_write(l, 'abc\n')
eq({ 'abc', 'abc', null_string }, lst2tbl(l))
end)
itp('writes string starting, ending and containing NL twice', function()
local l = list()
eq(0, encode_list_write(l, '\na\nb\n'))
encode_list_write(l, '\na\nb\n')
eq({ null_string, 'a', 'b', null_string }, lst2tbl(l))
eq(0, encode_list_write(l, '\na\nb\n'))
encode_list_write(l, '\na\nb\n')
eq({ null_string, 'a', 'b', null_string, 'a', 'b', null_string }, lst2tbl(l))
end)
itp('writes string starting, ending and containing NUL with NL between twice', function()
local l = list()
eq(0, encode_list_write(l, '\0\n\0\n\0'))
encode_list_write(l, '\0\n\0\n\0')
eq({ '\n', '\n', '\n' }, lst2tbl(l))
eq(0, encode_list_write(l, '\0\n\0\n\0'))
encode_list_write(l, '\0\n\0\n\0')
eq({ '\n', '\n', '\n\n', '\n', '\n' }, lst2tbl(l))
end)
itp('writes string starting, ending and containing NL with NUL between twice', function()
local l = list()
eq(0, encode_list_write(l, '\n\0\n\0\n'))
encode_list_write(l, '\n\0\n\0\n')
eq({ null_string, '\n', '\n', null_string }, lst2tbl(l))
eq(0, encode_list_write(l, '\n\0\n\0\n'))
encode_list_write(l, '\n\0\n\0\n')
eq({ null_string, '\n', '\n', null_string, '\n', '\n', null_string }, lst2tbl(l))
end)
itp('writes string containing a single NL twice', function()
local l = list()
eq(0, encode_list_write(l, '\n'))
encode_list_write(l, '\n')
eq({ null_string, null_string }, lst2tbl(l))
eq(0, encode_list_write(l, '\n'))
encode_list_write(l, '\n')
eq({ null_string, null_string, null_string }, lst2tbl(l))
end)
itp('writes string containing a few NLs twice', function()
local l = list()
eq(0, encode_list_write(l, '\n\n\n'))
encode_list_write(l, '\n\n\n')
eq({ null_string, null_string, null_string, null_string }, lst2tbl(l))
eq(0, encode_list_write(l, '\n\n\n'))
encode_list_write(l, '\n\n\n')
eq(
{ null_string, null_string, null_string, null_string, null_string, null_string, null_string },
lst2tbl(l)