Skip to content

src/box/tuple_hash.cc:317: uint32_t tuple_hash_field(uint32_t*, uint32_t*, const char**, field_type, coll*): Assertion `0' failed.  #10090

@ligurio

Description

@ligurio

Bug description

  • OS: Linux
  • OS Version: 22.04
  • Architecture: amd64

Tarantool 3.2.0-entrypoint-56-g05d03a1c58
Target: Linux-x86_64-Debug
Build options: cmake . -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BACKTRACE=TRUE
Compiler: GNU-11.4.0
C_FLAGS: -fexceptions -funwind-tables -fasynchronous-unwind-tables -fno-common -msse2 -Wformat -Wformat-security -Werror=format-security -fstack-protector-strong -fPIC -fmacro-prefix-map=/home/sergeyb/sources/MRG/tarantool=. -std=c11 -Wall -Wextra -Wno-gnu-alignof-expression -fno-gnu89-inline -Wno-cast-function-type -Werror -g -ggdb -O0
CXX_FLAGS: -fexceptions -funwind-tables -fasynchronous-unwind-tables -fno-common -msse2 -Wformat -Wformat-security -Werror=format-security -fstack-protector-strong -fPIC -fmacro-prefix-map=/home/sergeyb/sources/MRG/tarantool=. -std=c++11 -Wall -Wextra -Wno-invalid-offsetof -Wno-gnu-alignof-expression -Wno-cast-function-type -Werror -g -ggdb -O0

Steps to reproduce

Build Tarantool with enabled debug option:

$ cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug
$ cmake --build build/ --parallel

Execute Lua script vinyl.lua.

vinyl.lua
--[[ Тест для vinyl позволяет случайным образом выставлять error injections,
генерировать операции с индексами, генерировать операции с данными, настройки
vinyl в box.cfg. Все случайные операции и настройки зависят от seed, который
генерируется в самом начале теста.

Usage: taskset 0xef ./tarantool vinyl.lua
]]

local fiber = require('fiber')
local fun = require('fun')
local json = require('json')
local log = require('log')
local math = require('math')

local params = require('internal.argparse').parse(arg, {
    {'test_duration', 'number'},
    {'workers', 'number'},
    {'seed', 'number'},
    {'h', 'boolean'},
})

local function trace(event, line) -- luacheck: no unused
    local s = debug.getinfo(2).short_src
    if s == 'vinyl.lua' then
        log.info(s .. ":" .. line)
    end
end

-- https://www.lua.org/pil/23.2.html
if os.getenv('DEV') then
    debug.sethook(trace, "l")
end

if params.help or params.h then
    print([[

 Usage: taskset 0xef tarantool vinyl.lua [options]

 Options can be used with '--', followed by the value if it's not
 a boolean option. The options list with default values:

   workers <number, 50>          - number of fibers to run simultaneously
   test_duration <number, 2*60>  - test duration time (sec)
   seed                          - random seed
   help (same as -h)             - print this message
]])
    os.exit(0)
end

-- Number of workers.
local num_workers = params.workers or 50

-- Test duration time.
local test_duration = params.test_duration or 2*60

local seed = params.seed or os.time()
math.randomseed(seed)
log.info(string.format("Random seed: %d", seed))

local function dict_keys(t)
    assert(next(t) ~= nil)
    local keys = {}
    for k, _ in pairs(t) do
        table.insert(keys, k)
    end
    return keys
end

local function rand_char()
    return string.char(math.random(97, 97 + 25))
end

local function rand_string(length)
    length = length or 10
    return string.rep(rand_char(), length)
end

local function oneof(t)
    assert(type(t) == 'table')
    assert(next(t) ~= nil)

    local n = table.getn(t)
    local idx = math.random(1, n)
    return t[idx]
end

-- TODO: Obtain error injections dynamically:
-- box.error.injection.info()
-- box.error.injection.get('ERRINJ_SNAP_SKIP_ALL_ROWS')
-- box.error.injection.set('ERRINJ_SNAP_SKIP_ALL_ROWS', true)
-- Source: src/lib/core/errinj.h
local errinj_set = {
    ['ERRINJ_APPLIER_DESTROY_DELAY'] = 'boolean',
    ['ERRINJ_APPLIER_READ_TX_ROW_DELAY'] = 'boolean',
    ['ERRINJ_APPLIER_SLOW_ACK'] = 'boolean',
    ['ERRINJ_APPLIER_STOP_DELAY'] = 'boolean',
    ['ERRINJ_BUILD_INDEX'] = 'int',
    ['ERRINJ_BUILD_INDEX_DELAY'] = 'boolean',
    ['ERRINJ_BUILD_INDEX_ON_ROLLBACK_ALLOC'] = 'boolean',
    ['ERRINJ_BUILD_INDEX_TIMEOUT'] = 'double',
    ['ERRINJ_CHECK_FORMAT_DELAY'] = 'boolean',
    ['ERRINJ_COIO_SENDFILE_CHUNK'] = 'int',
    ['ERRINJ_COIO_WRITE_CHUNK'] = 'boolean',
    ['ERRINJ_DYN_MODULE_COUNT'] = 'int',
    ['ERRINJ_ENGINE_JOIN_DELAY'] = 'boolean',
    ['ERRINJ_FIBER_MADVISE'] = 'boolean',
    ['ERRINJ_FIBER_MPROTECT'] = 'int',
    ['ERRINJ_FLIGHTREC_RECREATE_RENAME'] = 'boolean',
    ['ERRINJ_FLIGHTREC_LOG_DELAY'] = 'double',
    ['ERRINJ_HTTPC_EXECUTE'] = 'boolean',
    ['ERRINJ_HTTP_RESPONSE_ADD_WAIT'] = 'boolean',
    ['ERRINJ_INDEX_ALLOC'] = 'boolean',
    ['ERRINJ_INDEX_RESERVE'] = 'boolean',
    ['ERRINJ_INDEX_ITERATOR_NEW'] = 'boolean',
    ['ERRINJ_HASH_INDEX_REPLACE'] = 'boolean',
    ['ERRINJ_IPROTO_CFG_LISTEN'] = 'boolean',
    ['ERRINJ_IPROTO_DISABLE_ID'] = 'boolean',
    ['ERRINJ_IPROTO_DISABLE_WATCH'] = 'boolean',
    ['ERRINJ_IPROTO_FLIP_FEATURE'] = 'int',
    ['ERRINJ_IPROTO_SET_VERSION'] = 'int',
    ['ERRINJ_IPROTO_TX_DELAY'] = 'boolean',
    ['ERRINJ_IPROTO_WRITE_ERROR_DELAY'] = 'boolean',
    ['ERRINJ_LOG_ROTATE'] = 'boolean',
    ['ERRINJ_MEMTX_DELAY_GC'] = 'boolean',
    ['ERRINJ_NETBOX_DISABLE_ID'] = 'boolean',
    ['ERRINJ_NETBOX_FLIP_FEATURE'] = 'int',
    ['ERRINJ_NETBOX_IO_DELAY'] = 'boolean',
    ['ERRINJ_NETBOX_IO_ERROR'] = 'boolean',
    ['ERRINJ_RAFT_WAIT_TERM_PERSISTED_DELAY'] = 'boolean',
    ['ERRINJ_RELAY_BREAK_LSN'] = 'int',
    ['ERRINJ_RELAY_EXIT_DELAY'] = 'double',
    ['ERRINJ_RELAY_FASTER_THAN_TX'] = 'boolean',
    ['ERRINJ_RELAY_FINAL_JOIN'] = 'boolean',
    ['ERRINJ_RELAY_FINAL_SLEEP'] = 'boolean',
    ['ERRINJ_RELAY_FROM_TX_DELAY'] = 'boolean',
    ['ERRINJ_RELAY_REPORT_INTERVAL'] = 'double',
    ['ERRINJ_RELAY_SEND_DELAY'] = 'boolean',
    ['ERRINJ_RELAY_TIMEOUT'] = 'double',
    ['ERRINJ_RELAY_WAL_START_DELAY'] = 'boolean',
    ['ERRINJ_REPLICASET_VCLOCK'] = 'boolean',
    ['ERRINJ_REPLICA_JOIN_DELAY'] = 'boolean',
    ['ERRINJ_SIGILL_MAIN_THREAD'] = 'boolean',
    ['ERRINJ_SIGILL_NONMAIN_THREAD'] = 'boolean',
    ['ERRINJ_SIO_READ_MAX'] = 'int',
    ['ERRINJ_SNAP_COMMIT_DELAY'] = 'boolean',
    ['ERRINJ_SNAP_COMMIT_FAIL'] = 'boolean',
    ['ERRINJ_SNAP_SKIP_ALL_ROWS'] = 'boolean',
    ['ERRINJ_SNAP_SKIP_DDL_ROWS'] = 'boolean',
    ['ERRINJ_SNAP_WRITE_DELAY'] = 'boolean',
    ['ERRINJ_SNAP_WRITE_CORRUPTED_INSERT_ROW'] = 'boolean',
    ['ERRINJ_SNAP_WRITE_INVALID_SYSTEM_ROW'] = 'boolean',
    ['ERRINJ_SNAP_WRITE_MISSING_SPACE_ROW'] = 'boolean',
    ['ERRINJ_SNAP_WRITE_TIMEOUT'] = 'double',
    ['ERRINJ_SNAP_WRITE_UNKNOWN_ROW_TYPE'] = 'boolean',
    ['ERRINJ_SPACE_UPGRADE_DELAY'] = 'boolean',
    ['ERRINJ_SWIM_FD_ONLY'] = 'boolean',
    ['ERRINJ_TESTING'] = 'boolean',
    ['ERRINJ_TUPLE_ALLOC'] = 'boolean',
    ['ERRINJ_TUPLE_FIELD'] = 'boolean',
    -- https://github.com/tarantool/tarantool/issues/10033
    -- ['ERRINJ_TUPLE_FIELD_COUNT_LIMIT'] = 'int',
    ['ERRINJ_TUPLE_FORMAT_COUNT'] = 'int',
    ['ERRINJ_TX_DELAY_PRIO_ENDPOINT'] = 'double',
    ['ERRINJ_TXN_COMMIT_ASYNC'] = 'boolean',
    ['ERRINJ_TXN_LIMBO_BEGIN_DELAY'] = 'boolean',
    ['ERRINJ_VYRUN_DATA_READ'] = 'boolean',
    ['ERRINJ_VY_COMPACTION_DELAY'] = 'boolean',
    ['ERRINJ_VY_DELAY_PK_LOOKUP'] = 'boolean',
    ['ERRINJ_VY_DUMP_DELAY'] = 'boolean',
    ['ERRINJ_VY_GC'] = 'boolean',
    ['ERRINJ_VY_INDEX_DUMP'] = 'int',
    ['ERRINJ_VY_INDEX_FILE_RENAME'] = 'boolean',
    ['ERRINJ_VY_LOG_FILE_RENAME'] = 'boolean',
    ['ERRINJ_VY_LOG_FLUSH'] = 'boolean',
    ['ERRINJ_VY_POINT_ITER_WAIT'] = 'boolean',
    ['ERRINJ_VY_QUOTA_DELAY'] = 'boolean',
    ['ERRINJ_VY_READ_PAGE'] = 'boolean',
    ['ERRINJ_VY_READ_PAGE_DELAY'] = 'boolean',
    ['ERRINJ_VY_READ_PAGE_TIMEOUT'] = 'double',
    ['ERRINJ_VY_READ_VIEW_MERGE_FAIL'] = 'boolean',
    ['ERRINJ_VY_RUN_DISCARD'] = 'boolean',
    ['ERRINJ_VY_RUN_FILE_RENAME'] = 'boolean',
    ['ERRINJ_VY_RUN_OPEN'] = 'int',
    ['ERRINJ_VY_RUN_WRITE'] = 'boolean',
    ['ERRINJ_VY_RUN_WRITE_DELAY'] = 'boolean',
    ['ERRINJ_VY_RUN_WRITE_STMT_TIMEOUT'] = 'double',
    ['ERRINJ_VY_SCHED_TIMEOUT'] = 'double',
    ['ERRINJ_VY_SQUASH_TIMEOUT'] = 'double',
    ['ERRINJ_VY_STMT_ALLOC'] = 'int',
    ['ERRINJ_VY_TASK_COMPLETE'] = 'boolean',
    ['ERRINJ_VY_WRITE_ITERATOR_START_FAIL'] = 'boolean',
    ['ERRINJ_WAIT_QUORUM_COUNT'] = 'int',
    ['ERRINJ_WAL_BREAK_LSN'] = 'int',
    ['ERRINJ_WAL_DELAY'] = 'boolean',
    ['ERRINJ_WAL_DELAY_COUNTDOWN'] = 'int',
    ['ERRINJ_WAL_FALLOCATE'] = 'int',
    ['ERRINJ_WAL_IO'] = 'boolean',
    ['ERRINJ_WAL_IO_COUNTDOWN'] = 'int',
    ['ERRINJ_WAL_ROTATE'] = 'boolean',
    ['ERRINJ_WAL_SYNC'] = 'boolean',
    ['ERRINJ_WAL_SYNC_DELAY'] = 'boolean',
    ['ERRINJ_WAL_WRITE'] = 'boolean',
    ['ERRINJ_WAL_WRITE_COUNT'] = 'int',
    ['ERRINJ_WAL_WRITE_DISK'] = 'boolean',
    ['ERRINJ_WAL_WRITE_EOF'] = 'boolean',
    ['ERRINJ_WAL_WRITE_PARTIAL'] = 'int',
    ['ERRINJ_XLOG_GARBAGE'] = 'boolean',
    ['ERRINJ_XLOG_META'] = 'boolean',
    ['ERRINJ_XLOG_READ'] = 'int',
    ['ERRINJ_XLOG_RENAME_DELAY'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_CORRUPTED_BODY'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_CORRUPTED_HEADER'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_INVALID_BODY'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_INVALID_HEADER'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_INVALID_KEY'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_INVALID_VALUE'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_UNKNOWN_KEY'] = 'boolean',
    ['ERRINJ_XLOG_WRITE_UNKNOWN_TYPE'] = 'boolean',
}

-- Forward declaration.
local index_create_op

local tx_op = {
    ['TX_BEGIN'] = function() box.begin() end,
    ['TX_COMMIT'] = function() box.rollback() end,
    ['TX_ROLLBACK'] = function() box.commit() end,
}

local tarantool_type = {
    -- 'any',
    -- 'array',
    'boolean',
    -- 'decimal',
    -- 'datetime',
    'double',
    'integer',
    -- 'map',
    'number',
    -- 'scalar',
    'string',
    -- 'unsigned',
    -- 'uuid',
}

-- The name value may be any string, provided that two fields
-- do not have the same name.
-- The type value may be any of allowed types:
-- any | unsigned | string | integer | number | varbinary |
-- boolean | double | decimal | uuid | array | map | scalar,
-- but for creating an index use only indexed fields;
-- (Optional) The is_nullable boolean value specifies whether
-- nil can be used as a field value. See also: key_part.is_nullable.
-- (Optional) The collation string value specifies the collation
-- used to compare field values. See also: key_part.collation.
-- (Optional) The constraint table specifies the constraints that
-- the field value must satisfy.
-- (Optional) The foreign_key table specifies the foreign keys
-- for the field.
local function random_space_format()
    local space_format = {}
    local min_num_fields = 3
    local max_num_fields = 10
    local num_fields = math.random(min_num_fields, max_num_fields)
    for i = 1, num_fields do
        table.insert(space_format, {
            name =('name_%d'):format(i),
            type = oneof(tarantool_type),
        })
    end
    return space_format
end

-- Iterator types for TREE indexes.
-- https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_index/pairs/#box-index-iterator-types
local iter_type = {
    'ALL',
    'EQ',
    'GE',
    'GT',
    'LE',
    'LT',
    'REQ',
}

local function select_op(space, key)
    local select_opts = {
        iterator = oneof(iter_type),
        -- The maximum number of tuples.
        limit = math.random(100, 500),
        -- The number of tuples to skip.
        offset = math.random(100),
        -- A tuple or the position of a tuple (tuple_pos) after
        -- which select starts the search.
        after = box.NULL,
        -- If true, the select method returns the position of
        -- the last selected tuple as the second value.
        fetch_pos = oneof({true, false}),
    }
    space:select(key, select_opts)
end

local function delete_op(space, tuple)
    space:delete(tuple)
end

local function insert_op(space, tuple)
    space:insert(tuple)
end

local tuple_op = {
    '+', -- numeric.
    '-', -- numeric.
    '&', -- numeric.
    '|', -- numeric.
    '^', -- numeric.
    '!', -- for insertion of a new field.
    '#', -- for deletion.
    '=', -- for assignment.
    -- ':', for string splice.
}

local function upsert_op(space, tuple, spec)
    space:upsert(tuple, spec)
end

local function update_op(space, key, spec)
    space:update(key, spec)
end

local function replace_op(space, tuple)
    space:replace(tuple)
end

local function bsize_op(space)
    space:bsize()
end

local function len_op(space)
    space:len()
end

local function format_op(space)
    -- https://www.tarantool.io/ru/doc/latest/reference/reference_lua/box_space/format/
    local space_format = random_space_format()
    space:format(space_format)
end

local dml_ops = {
    DELETE_OP = delete_op,
    INSERT_OP = insert_op,
    SELECT_OP = select_op,
    REPLACE_OP = replace_op,
    UPDATE_OP = update_op,
    UPSERT_OP = upsert_op,
    BSIZE_OP = bsize_op,
    LEN_OP = len_op,
    FORMAT_OP = format_op,
}

local function setup(engine)
    log.info("SETUP")
    local engine_name = engine or oneof({'vinyl', 'memtx'})
    assert(engine_name == 'memtx' or engine_name == 'vinyl')
    -- TODO: https://www.tarantool.io/en/doc/2.3/reference/configuration/
    box.cfg{
        memtx_memory = 1024*1024,
        vinyl_cache = math.random(0, 10),
        vinyl_bloom_fpr = math.random(50) / 100,
        vinyl_max_tuple_size = math.random(0, 100000),
        vinyl_memory = 10*1024*1024,
        -- vinyl_page_size = math.random(1, 10),
        -- vinyl_range_size = math.random(1, 10),
        vinyl_run_size_ratio = math.random(2, 5),
        vinyl_run_count_per_level = math.random(1, 10),
        vinyl_read_threads = math.random(2, 10),
        vinyl_write_threads = math.random(2, 10),
        vinyl_timeout = math.random(1, 5),
        wal_mode = oneof({'write', 'fsync'}),
        wal_max_size = math.random(1024 * 1024 * 1024),
        checkpoint_interval = math.random(60),
        checkpoint_count = math.random(5),
        checkpoint_wal_threshold = math.random(1024),
    }
    log.info('FINISH BOX.CFG')

    log.info('CREATE A SPACE')
    local space_opts = {
        engine = engine_name,
        is_local = oneof({true, false}),
        if_not_exists = oneof({true, false}),
        field_count = 0,
        format = random_space_format(),
        -- temporary = oneof({true, false}),
        -- is_sync = oneof({true, false}),
        -- TODO: constraint =
        -- TODO: foreign_key =
    }
    log.info(space_opts)
    local space_name = ('test_%d'):format(math.random(100))
    local space = box.schema.space.create(space_name, space_opts)
    index_create_op(space)
    index_create_op(space)
    log.info('FINISH SETUP')
    return space
end

local function cleanup()
   log.info("CLEANUP")
   os.execute('rm -rf *.snap *.xlog *.vylog 51*')
end

local function teardown(space)
   log.info("TEARDOWN")
   space:drop()
   cleanup()
end

-- https://www.tarantool.io/en/doc/latest/concepts/data_model/indexes/
local function index_opts(space)
    assert(space ~= nil)
    -- TODO: generate random 'parts' by specified space format.
    local opts = {
        unique = oneof({true, false}),
        if_not_exists = false,
        -- sequence,
        -- func,
        -- page_size,
        -- range_size,
        -- run_count_per_level,
        -- run_size_ratio,
    }

    -- TODO: RTREE, BITSET
    opts.type = oneof({'TREE', 'HASH'})
    if space.engine == 'memtx' then
        opts.hint = oneof({true, false})
    end

    if space.engine == 'vinyl' then
        opts.bloom_fpr = math.random(50) / 100
    end

    opts.parts = {}
    local space_format = space:format()
    local n_parts = math.random(table.getn(space_format))
    for i = 1, n_parts do
        local field_name = space_format[i].name
        table.insert(opts.parts, { field_name })
    end

    if opts.type == 'RTREE' then
        opts.dimension = math.random(10)
        opts.distance = oneof({'euclid', 'manhattan'})
    end

    return opts
end

-- local function counter()
--     local i = 0
--     return function() return i + 1 end
-- end

function index_create_op(space)
    local idx_name = 'idx_' .. math.random(100)
    if space.index[idx_name] ~= nil then
        space.index[idx_name]:drop()
    end
    local opts = index_opts(space)
    -- FIXME
    opts.type = 'TREE'
    local ok, err = pcall(space.create_index, space, idx_name, opts)
    if ok ~= true then
        local msg = ('ERROR: %s (%s)'):format(err, json.encode(opts))
        log.info(msg)
    end
end

local function index_drop_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:drop() end
end

local function index_alter_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    local opts = index_opts(space)
    -- Option is not relevant.
    opts.if_not_exists = nil
    if idx ~= nil then idx:alter(opts) end
end

local function index_compact_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:compact() end
end

local function index_max_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:max() end
end

local function index_min_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:min() end
end

local function index_random_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil and
       idx.type ~= 'TREE' then
        idx:random()
    end
end

local function index_rename_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    local space_name = rand_string()
    if idx ~= nil then idx:rename(space_name) end
end

local function index_stat_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:stat() end
end

local function index_get_op(space, key)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:get(key) end
end

local function index_select_op(space, key)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:select(key) end
end

local function index_count_op(space)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:count() end
end

local function index_update_op(space, key, spec)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:update(key, spec) end
end

local function index_delete_op(space, tuple)
    if not space.enabled then return end
    local idx = oneof(space.index)
    if idx ~= nil then idx:delete(tuple) end
end

local ddl_ops = {
    INDEX_ALTER_OP = index_alter_op,
    INDEX_COMPACT_OP = index_compact_op,
    INDEX_CREATE_OP = index_create_op,
    INDEX_DROP_OP = index_drop_op,
    INDEX_GET_OP = index_get_op,
    INDEX_SELECT_OP = index_select_op,
    INDEX_MIN_OP = index_min_op,
    INDEX_MAX_OP = index_max_op,
    INDEX_RANDOM_OP = index_random_op,
    INDEX_COUNT_OP = index_count_op,
    INDEX_UPDATE_OP = index_update_op,
    INDEX_DELETE_OP = index_delete_op,
    INDEX_RENAME_OP = index_rename_op,
    INDEX_STAT_OP = index_stat_op,
}

local function set_err_injection()
    local errinj_name = oneof(dict_keys(errinj_set))
    local t = errinj_set[errinj_name]

    local errinj_val_enable = true
    local errinj_val_disable = false
    if t == 'double' then
        errinj_val_enable = math.random(0, 50)
        errinj_val_disable = 0
    end
    if t == 'int' then
        errinj_val_enable = math.random(0, 50)
        errinj_val_disable = -1
    end

    local pause_time = math.random(1, 10)

    log.info(string.format("ENABLE RANDOM ERROR INJECTION: %s -> %s",
                           errinj_name, tostring(errinj_val_enable)))
    local ok, err
    ok, err = pcall(box.error.injection.set, errinj_name, errinj_val_enable)
    if ok ~= true then
        log.info('ERROR: ' .. err)
    end
    fiber.sleep(pause_time)
    log.info(string.format("DISABLE RANDOM ERROR INJECTION: %s -> %s",
                           errinj_name, tostring(errinj_val_disable)))
    ok, err = pcall(box.error.injection.set, errinj_name, errinj_val_disable)
    if ok ~= true then
        log.info('ERROR: ' .. err)
    end
end

-- TODO: support is_nullable.
local function random_tuple(space_format)
    local tuple = {}
    for _, field in ipairs(space_format) do
        local v
        if field.type == 'number' or
           field.type == 'integer' or
           field.type == 'unsigned' then
            v = math.floor(math.random() * 10^12)
        elseif field.type == 'boolean' then
            v = oneof({true, false})
        elseif field.type == 'double' then
            v = math.random() * 10^12
        elseif field.type == 'string' then
            v = rand_string()
        end
        table.insert(tuple, v)
    end

    return tuple
end

local function random_spec()
    -- TODO
end

local function random_key(space)
    local pk = space.index[0]
    log.info(pk)
    assert(pk)
    local parts = pk.parts
    local key = {}
    for _, field in ipairs(parts) do
        local v
        if field.type == 'number' or
           field.type == 'integer' or
           field.type == 'unsigned' then
            v = math.floor(math.random() * 10^12)
        elseif field.type == 'boolean' then
            v = oneof({true, false})
        elseif field.type == 'double' then
            v = math.random() * 10^12
        elseif field.type == 'string' then
            v = rand_string()
        end
        table.insert(key, v)
    end
    return key
end

-- TODO:
-- - tx_op
-- - ddl_ops
-- - errinj_set
-- - snapshots
local function tarantool_ops_gen(space)
    return fun.cycle(fun.iter({
        { 'INSERT_OP', random_tuple(space:format()) },
        { 'SELECT_OP', random_key(space) },
        { 'REPLACE_OP', random_tuple(space:format()) },
        { 'UPSERT_OP', random_tuple(space:format()), random_spec() },
        { 'UPDATE_OP', random_tuple(space:format()), random_spec() },
        { 'DELETE_OP', random_key(space) },
        { 'BSIZE_OP' },
        { 'LEN_OP' },
        { 'FORMAT_OP' },
        { 'TX_BEGIN' },
        { 'TX_COMMIT' },
        { 'TX_ROLLBACK' },
    }))
end

local function apply_op(space, op)
    log.info(op)
    local op_name = op[1]
    local args = {}
    if op_name == 'INSERT_OP' then
        local func = dml_ops[op_name]
        local tuple = op[2]
        args = {func, space, tuple}
    elseif op_name == 'SELECT_OP' then
        local func = dml_ops[op_name]
        local key = op[2]
        args = {func, space, key}
    elseif op_name == 'REPLACE_OP' then
        local func = dml_ops[op_name]
        local tuple = op[2]
        args = {func, space, tuple}
    elseif op_name == 'DELETE_OP' then
        local func = dml_ops[op_name]
        local key = op[2]
        args = {func, space, key}
    elseif op_name == 'UPDATE_OP' then
        -- TODO
        return
    elseif op_name == 'UPSERT_OP' then
        -- TODO
        return
    elseif op_name == 'BSIZE_OP' then
        local func = dml_ops[op_name]
        args = {func, space}
    elseif op_name == 'LEN_OP' then
        local func = dml_ops[op_name]
        args = {func, space}
    elseif op_name == 'FORMAT_OP' then
        -- TODO
        return
    elseif op_name == 'TX_BEGIN' then
        local func = tx_op[op_name]
        args = {func}
    elseif op_name == 'TX_COMMIT' then
        local func = tx_op[op_name]
        args = {func}
    elseif op_name == 'TX_ROLLBACK' then
        local func = tx_op[op_name]
        args = {func}
    end
    local ok, err = pcall(unpack(args))
    if ok ~= true then
        log.info('ERROR: ' .. err)
        -- log.info(space.index[0].parts)
    end
end

local function worker_func(space)
    local start = os.clock()
    for _, operation in tarantool_ops_gen(space) do
        if os.clock() - start >= test_duration then
            break
        end
        apply_op(space, operation)
        fiber.sleep(0.001)
    end
end

local function run_test()
    local fibers = {}

    cleanup()
    local space = setup('vinyl')

    local f
    for i = 1, num_workers do
        f = fiber.new(worker_func, space)
        f:set_joinable(true)
        f:name('WORKER #' .. i)
        table.insert(fibers, f)
    end

    for _, fb in ipairs(fibers) do
        local ok, errmsg = fiber.join(fb)
        if not ok then
            log.info('ERROR: ' .. errmsg)
        end
    end

    teardown(space)
end

run_test()
os.exit(0)

Just type taskset 0xef ./tarantool vinyl.lua --workers 500 --test_duration $((5*60)) --seed 1717516396.

$ taskset 0xef ./tarantool vinyl.lua --workers 500 --test_duration $((5*60))                                 
Random seed: 1717516396                                                                                                                            
CLEANUP                                                                                                                                            
SETUP                                                                                                                                              
2024-06-04 18:53:16.808 [348994] main/104/vinyl.lua I> Tarantool 3.2.0-entrypoint-56-g05d03a1c58 Linux-x86_64-Debug                                
2024-06-04 18:53:16.808 [348994] main/104/vinyl.lua I> log level 5 (INFO)                                                                          
2024-06-04 18:53:16.808 [348994] main/104/vinyl.lua I> wal/engine cleanup is paused                                                                
2024-06-04 18:53:16.808 [348994] main/104/vinyl.lua I> mapping 67108864 bytes for memtx tuple arena...                                             
2024-06-04 18:53:16.808 [348994] main/104/vinyl.lua I> Actual slab_alloc_factor calculated on the basis of desired slab_alloc_factor = 1.044274    
2024-06-04 18:53:16.809 [348994] main/104/vinyl.lua I> mapping 16777216 bytes for vinyl tuple arena...                                             
2024-06-04 18:53:16.816 [348994] main/104/vinyl.lua I> update replication_synchro_quorum = 1                                                       
2024-06-04 18:53:16.816 [348994] main/104/vinyl.lua I> instance uuid e7e20095-e96e-4047-8eb3-495f62cf7d70                                          
2024-06-04 18:53:16.816 [348994] main/104/vinyl.lua I> initializing an empty data directory                                                        
2024-06-04 18:53:16.838 [348994] main/104/vinyl.lua I> assigned id 1 to replica e7e20095-e96e-4047-8eb3-495f62cf7d70                               
2024-06-04 18:53:16.838 [348994] main/104/vinyl.lua I> update replication_synchro_quorum = 1                                                       
2024-06-04 18:53:16.838 [348994] main/104/vinyl.lua I> replicaset uuid 6e9fbc24-9703-47e0-890e-8e5adfd83de7                                        
2024-06-04 18:53:16.840 [348994] snapshot/101/main I> saving snapshot `./00000000000000000000.snap.inprogress'
2024-06-04 18:53:16.851 [348994] snapshot/101/main I> done
2024-06-04 18:53:16.851 [348994] main/104/vinyl.lua I> ready to accept requests                                                                    
2024-06-04 18:53:16.851 [348994] main/105/gc I> wal/engine cleanup is resumed                                                                      
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'memtx_memory' configuration option to 1048576                             
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'vinyl_memory' configuration option to 10485760                            
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'vinyl_cache' configuration option to 7                                    
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'vinyl_timeout' configuration option to 3                                  
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'vinyl_max_tuple_size' configuration option to 39133                       
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'checkpoint_count' configuration option to 3                               
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'checkpoint_wal_threshold' configuration option to 645                     
2024-06-04 18:53:16.852 [348994] main/104/vinyl.lua/box.load_cfg I> set 'checkpoint_interval' configuration option to 12                           
2024-06-04 18:53:16.853 [348994] main/104/vinyl.lua/vinyl I> FINISH BOX.CFG                                                                        
2024-06-04 18:53:16.853 [348994] main/104/vinyl.lua/vinyl I> CREATE A SPACE                                                                        
2024-06-04 18:53:16.853 [348994] main/104/vinyl.lua/vinyl I> {"format":[{"name":"name_1","type":"number"},{"name":"name_2","type":"double"},{"name"
:"name_3","type":"string"},{"name":"name_4","type":"double"},{"name":"name_5","type":"integer"},{"name":"name_6","type":"integer"},{"name":"name_7"
,"type":"string"}],"if_not_exists":true,"is_local":false,"field_count":0,"engine":"vinyl"}                                                         
2024-06-04 18:53:16.853 [348994] main/106/checkpoint_daemon I> scheduled next checkpoint for Tue Jun  4 18:53:34 2024
2024-06-04 18:53:16.861 [348994] main/104/vinyl.lua/vinyl I> FINISH SETUP                             
2024-06-04 18:53:17.004 [348994] main/114/WORKER #1/vinyl I> {"unique":true,"parts":[{"fieldno":1,"sort_order":"asc","type":"number","exclude_null"
:false,"is_nullable":false},{"fieldno":2,"sort_order":"asc","type":"double","exclude_null":false,"is_nullable":false},{"fieldno":3,"sort_order":"as
c","type":"string","exclude_null":false,"is_nullable":false},{"fieldno":4,"sort_order":"asc","type":"double","exclude_null":false,"is_nullable":fal
se}],"options":{"page_size":8192,"run_count_per_level":2,"run_size_ratio":3,"bloom_fpr":0.01},"space_id":512,"type":"TREE","name":"idx_89","id":0}
2024-06-04 18:53:17.004 [348994] main/114/WORKER #1/vinyl I> {"unique":true,"parts":[{"fieldno":1,"sort_order":"asc","type":"number","exclude_null"
:false,"is_nullable":false},{"fieldno":2,"sort_order":"asc","type":"double","exclude_null":false,"is_nullable":false},{"fieldno":3,"sort_order":"as
c","type":"string","exclude_null":false,"is_nullable":false},{"fieldno":4,"sort_order":"asc","type":"double","exclude_null":false,"is_nullable":fal
se}],"options":{"page_size":8192,"run_count_per_level":2,"run_size_ratio":3,"bloom_fpr":0.01},"space_id":512,"type":"TREE","name":"idx_89","id":0}
2024-06-04 18:53:17.004 [348994] main/114/WORKER #1/vinyl I> ["INSERT_OP",[171574381066,347929122126.29,"zzzzzzzzzz",473129152178.67,213542720760,3
64430685019,"oooooooooo"]]

<snipped>

2024-06-04 18:53:17.930 [348994] main/323/WORKER #210/vinyl I> ["DELETE_OP",[608808197894,192275516720.45,"iiiiiiiiii",352910343117.08]]           
2024-06-04 18:53:17.930 [348994] main/560/WORKER #447/vinyl I> ["REPLACE_OP",[658131881835,319783564572.54,"cccccccccc",822171900162.62,22024121121
1,912430631936,"ssssssssss"]]                                                                                                                      
2024-06-04 18:53:17.930 [348994] main/366/WORKER #253/vinyl I> ["REPLACE_OP",[481642672373,759155310468.41,"qqqqqqqqqq",217652137616.5,196411227098
,344141458381,"iiiiiiiiii"]]                                                                                                                       
2024-06-04 18:53:17.933 [348994] main/108/vinyl.scheduler I> 512/1: started compacting range (-inf..inf), runs 2/2                                 
2024-06-04 18:53:17.934 [348994] main/599/WORKER #486/vinyl I> ["REPLACE_OP",[814869088513,556918596024.23,"iiiiiiiiii",232274804489.89,60593788813
4,223193337946,"tttttttttt"]]                                                                                                                      
2024-06-04 18:53:17.934 [348994] main/220/WORKER #107/vinyl I> ["DELETE_OP",[439013051616,632317800696.56,"uuuuuuuuuu",324764759111.48]]           
2024-06-04 18:53:17.934 [348994] main/234/WORKER #121/vinyl I> ["DELETE_OP",[703005905592,48860511118.638,"ffffffffff",287148503908.69]]           
2024-06-04 18:53:17.934 [348994] main/126/WORKER #13/vinyl I> ["DELETE_OP",[681913360020,26496878612.665,"yyyyyyyyyy",914873148846.31]]            
2024-06-04 18:53:17.934 [348994] vinyl.compaction.0/102/task I> writing `./512/1/00000000000000000011.run'                                         
2024-06-04 18:53:17.934 [348994] main/574/WORKER #461/vinyl I> ["DELETE_OP",[822119347645,671470208714.58,"ssssssssss",902115976198.12]]           
2024-06-04 18:53:17.934 [348994] main/336/WORKER #223/vinyl I> ["DELETE_OP",[19861597823,814975273290.99,"dddddddddd",153616701864.34]]            
tarantool: ./src/box/tuple_hash.cc:317: uint32_t tuple_hash_field(uint32_t*, uint32_t*, const char**, field_type, coll*): Assertion `0' failed.    
2024-06-04 18:53:17.934 [348994] main/459/WORKER #346/vinyl I> ["DELETE_OP",[19402349170,344888453368.97,"uuuuuuuuuu",692763000981.61]]            
Aborted (core dumped)

Actual behavior

coredump and tarantool binary: gh-10090-core.zip

(gdb) bt
#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=123397791020608) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=6, threadid=123397791020608) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=123397791020608, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#3  0x0000703b41242476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4  0x0000703b412287f3 in __GI_abort () at ./stdlib/abort.c:79
#5  0x0000703b4122871b in __assert_fail_base (fmt=0x703b413dd130 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x5a29adc2a39d "0", 
    file=0x5a29adc2abba "./src/box/tuple_hash.cc", line=317, function=<optimized out>) at ./assert/assert.c:92
#6  0x0000703b41239e96 in __GI___assert_fail (assertion=0x5a29adc2a39d "0", file=0x5a29adc2abba "./src/box/tuple_hash.cc", line=317, 
    function=0x5a29adc2ac40 "uint32_t tuple_hash_field(uint32_t*, uint32_t*, const char**, field_type, coll*)") at ./assert/assert.c:101
#7  0x00005a29ad8d7a87 in tuple_hash_field (ph1=0x703ac8a7f344, pcarry=0x703ac8a7f348, field=0x703ac8a7f330, type=FIELD_TYPE_DOUBLE, coll=0x0)
    at /home/sergeyb/sources/MRG/tarantool/src/box/tuple_hash.cc:317
#8  0x00005a29ad8c8332 in tuple_bloom_builder_add_key (builder=0x703ac00104d0, key=0x703ac000f4b7 "\252mmmmmmmmmm\313B,\254\374jA{\264", 
    part_count=4, key_def=0x5a29aefd3f80) at /home/sergeyb/sources/MRG/tarantool/src/box/tuple_bloom.c:143
#9  0x00005a29ad3b1c02 in vy_bloom_builder_add (builder=0x703ac00104d0, entry=..., key_def=0x5a29aefd3f80)
    at /home/sergeyb/sources/MRG/tarantool/src/box/vy_stmt.c:570
#10 0x00005a29ad3c8621 in vy_run_writer_write_to_page (writer=0x703ac8a7f4e0, entry=...)
    at /home/sergeyb/sources/MRG/tarantool/src/box/vy_run.c:2217
#11 0x00005a29ad3c8ba9 in vy_run_writer_append_stmt (writer=0x703ac8a7f4e0, entry=...)
    at /home/sergeyb/sources/MRG/tarantool/src/box/vy_run.c:2291
#12 0x00005a29ad40eca4 in vy_task_write_run (task=0x5a29afa9e090, no_compression=false)
    at /home/sergeyb/sources/MRG/tarantool/src/box/vy_scheduler.c:1132
#13 0x00005a29ad4102f8 in vy_task_compaction_execute (task=0x5a29afa9e090) at /home/sergeyb/sources/MRG/tarantool/src/box/vy_scheduler.c:1485
#14 0x00005a29ad41176b in vy_task_f (va=0x703ac8810440) at /home/sergeyb/sources/MRG/tarantool/src/box/vy_scheduler.c:1795
#15 0x00005a29ad324147 in fiber_cxx_invoke(fiber_func, typedef __va_list_tag __va_list_tag *) (f=0x5a29ad4115ae <vy_task_f>, ap=0x703ac8810440)
    at /home/sergeyb/sources/MRG/tarantool/src/lib/core/fiber.h:1311
#16 0x00005a29ad5f6bd1 in fiber_loop (data=0x0) at /home/sergeyb/sources/MRG/tarantool/src/lib/core/fiber.c:1159
#17 0x00005a29ad9e00f6 in coro_init () at /home/sergeyb/sources/MRG/tarantool/third_party/coro/coro.c:108
(gdb) 

relevant source code:

unreachable();

Expected behavior

no crash

Metadata

Metadata

Assignees

Labels

2.11Target is 2.11 and all newer release/master branches3.1Target is 3.1 and all newer release/master branchesbugSomething isn't workingcrashfuzzingvinyl

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions