intermittent segfault with small database with multiple column families

[philmes]

Expected behavior

When closing and reopening a database and reading a value from a column family, it should not segfault.

Actual behavior

It segfaults.

Steps to reproduce the behavior

RocksDB 8.11.3

This small C program intermittently segfaults (backtrace below)

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>

#include "rocksdb/c.h"

const char DBPath[] = "/tmp/crash-test";
const char Key[] = "k";
const char Value[] = "v";

void setup() {
    rocksdb_t *db;
    rocksdb_options_t *options = rocksdb_options_create();
    rocksdb_options_set_create_if_missing(options, 1);
    rocksdb_options_set_create_missing_column_families(options, true);

    rocksdb_column_family_handle_t* cfHandles[2];

    char* cfNames[2];
    cfNames[0] = "default";
    cfNames[1] = "metadata";

    rocksdb_options_t* cfOpts[2];
    cfOpts[0] = rocksdb_options_create();
    cfOpts[1] = rocksdb_options_create();

    // open DB
    char *err = NULL;
    db = rocksdb_open_column_families(options, DBPath, 2, (const char**)cfNames,
                                      (const rocksdb_options_t**)cfOpts,
                                      cfHandles, &err
    );
    assert(!err);

    // Put key-value
    rocksdb_writeoptions_t *writeoptions = rocksdb_writeoptions_create();

    rocksdb_put_cf(db, writeoptions, cfHandles[1], Key, strlen(Key), Value, strlen(Value)+1, &err);
    assert(!err);

    rocksdb_readoptions_t *readoptions = rocksdb_readoptions_create();
    size_t len;

    char *returned_value = rocksdb_get_cf(db, readoptions, cfHandles[1], Key, strlen(Key), &len, &err);
    assert(!err);
    assert(strcmp(returned_value, Value) == 0);
    free(returned_value);

    rocksdb_column_family_handle_destroy(cfHandles[0]);
    rocksdb_column_family_handle_destroy(cfHandles[1]);

    rocksdb_close(db);
}

void trigger() {
    rocksdb_t *db;
    rocksdb_options_t *options = rocksdb_options_create();
    rocksdb_options_set_create_if_missing(options, 1);
    rocksdb_options_set_create_missing_column_families(options, true);

    rocksdb_column_family_handle_t* cfHandles[2];

    char* cfNames[2];
    cfNames[0] = "default";
    cfNames[1] = "metadata";

    rocksdb_options_t* cfOpts[2];
    cfOpts[0] = rocksdb_options_create();
    cfOpts[1] = rocksdb_options_create();

    // open DB
    char *err = NULL;
    db = rocksdb_open_column_families(options, DBPath, 2, (const char**)cfNames,
                                      (const rocksdb_options_t**)cfOpts,
                                      cfHandles, &err
    );
    assert(!err);

    rocksdb_readoptions_t *readoptions = rocksdb_readoptions_create();
    size_t len;

    char *returned_value = rocksdb_get_cf(db, readoptions, cfHandles[1], Key, strlen(Key), &len, &err);
    assert(!err);
    assert(strcmp(returned_value, Value) == 0);
    free(returned_value);

    rocksdb_close(db);
}

int main(int argc, char *argv[]) {
    setup();
    trigger();

   return 0;
}

#1  __malloc_usable_size (m=0x555555dd2e40) at malloc.c:5252
#2  0x00007ffff7ab97ab in rocksdb::Block::ApproximateMemoryUsage (this=this@entry=0x555555dd2e40) at table/block_based/block.cc:1328
#3  0x00007ffff7b02d08 in rocksdb::BlockBasedTable::PutDataBlockToCache<rocksdb::Block_kData> (this=this@entry=0x555555dd7c20, cache_key=..., block_cache=..., out_parsed_block=out_parsed_block@entry=0x7fffffffd060, 
    uncompressed_block_contents=..., compressed_block_contents=..., block_comp_type=rocksdb::kNoCompression, uncompression_dict=..., memory_allocator=0x0, get_context=0x7fffffffdaf0)
    at table/block_based/block_based_table_reader.cc:1381
#4  0x00007ffff7b03b9d in rocksdb::BlockBasedTable::MaybeReadBlockAndLoadToCache<rocksdb::Block_kData> (this=this@entry=0x555555dd7c20, prefetch_buffer=prefetch_buffer@entry=0x0, ro=..., handle=..., uncompression_dict=..., 
    for_compaction=false, out_parsed_block=0x7fffffffd060, get_context=0x7fffffffdaf0, lookup_context=0x7fffffffd310, contents=0x0, async_read=false, use_block_cache_for_lookup=true)
    at table/block_based/block_based_table_reader.cc:1662
#5  0x00007ffff7b0413d in rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::Block_kData> (this=this@entry=0x555555dd7c20, prefetch_buffer=prefetch_buffer@entry=0x0, ro=..., handle=..., uncompression_dict=..., 
    out_parsed_block=0x7fffffffd060, get_context=0x7fffffffdaf0, lookup_context=0x7fffffffd310, for_compaction=false, use_cache=true, async_read=false, use_block_cache_for_lookup=true)
    at table/block_based/block_based_table_reader.cc:1804
#6  0x00007ffff7b0474f in rocksdb::BlockBasedTable::NewDataBlockIterator<rocksdb::DataBlockIter> (this=this@entry=0x555555dd7c20, ro=..., handle=..., input_iter=input_iter@entry=0x7fffffffd520, 
    block_type=block_type@entry=rocksdb::BlockType::kData, get_context=get_context@entry=0x7fffffffdaf0, lookup_context=0x7fffffffd310, prefetch_buffer=0x0, for_compaction=false, async_read=false, s=..., 
    use_block_cache_for_lookup=true) at ./table/block_based/block_based_table_reader_impl.h:89
#7  0x00007ffff7af8b2a in rocksdb::BlockBasedTable::Get (this=0x555555dd7c20, read_options=..., key=..., get_context=0x7fffffffdaf0, prefix_extractor=<optimized out>, skip_filters=false)
    at table/block_based/block_based_table_reader.cc:2268
#8  0x00007ffff79437d0 in rocksdb::TableCache::Get (this=this@entry=0x555555d9c120, options=..., internal_comparator=..., file_meta=..., k=..., get_context=0x7fffffffdaf0, block_protection_bytes_per_key=0 '\000', 
--Type <RET> for more, q to quit, c to continue without paging--
    prefix_extractor=std::shared_ptr<const rocksdb::SliceTransform> (empty) = {...}, file_read_hist=0x555555e46000, skip_filters=false, level=0, max_file_size_for_l0_meta_pin=100663296) at db/table_cache.cc:489
#9  0x00007ffff797d6eb in rocksdb::Version::Get (this=0x555555e27400, read_options=..., k=..., value=0x7fffffffe2d0, columns=0x0, timestamp=timestamp@entry=0x0, status=0x7fffffffde10, merge_context=0x7fffffffde50, 
    max_covering_tombstone_seq=0x7fffffffde00, pinned_iters_mgr=0x7fffffffdf40, value_found=0x0, key_exists=0x0, seq=0x0, callback=0x0, is_blob=0x0, do_merge=true) at db/version_set.cc:2437
#10 0x00007ffff7808ca4 in rocksdb::DBImpl::GetImpl (this=0x555555de6000, read_options=..., key=..., get_impl_options=...) at db/db_impl/db_impl.cc:2394
#11 0x00007ffff780030e in rocksdb::DBImpl::GetImpl (this=this@entry=0x555555de6000, read_options=..., column_family=column_family@entry=0x555555eb4440, key=..., value=value@entry=0x7fffffffe2d0, timestamp=0x0)
    at db/db_impl/db_impl.cc:2066
#12 0x00007ffff78005c8 in rocksdb::DBImpl::Get (this=0x555555de6000, _read_options=..., column_family=0x555555eb4440, key=..., value=0x7fffffffe2d0, timestamp=0x0) at db/db_impl/db_impl.cc:2054
#13 0x00007ffff77f8794 in rocksdb::DBImpl::Get (this=<optimized out>, read_options=..., column_family=<optimized out>, key=..., value=<optimized out>) at db/db_impl/db_impl.cc:2026
#14 0x00007ffff777e2fb in rocksdb::DB::Get (this=0x555555de6000, options=..., column_family=0x555555eb4440, key=..., value=0x7fffffffe3a0) at ./include/rocksdb/db.h:562
#15 0x00007ffff77604a9 in rocksdb_get_cf (db=<optimized out>, options=<optimized out>, column_family=<optimized out>, key=<optimized out>, keylen=<optimized out>, vallen=0x7fffffffe418, errptr=0x7fffffffe410) at db/c.cc:1313
#16 0x0000555555555556 in trigger () at /home/phil/source/rocksdb-crash-test/main.c:83
#17 0x000055555555565b in main (argc=2, argv=0x7fffffffe5b8) at /home/phil/source/rocksdb-crash-test/main.c:99

[philmes]

Should also mention that this happens on Linux (Ubuntu 22.04 and an up to date Arch). Doesn't trigger on OSX

[philmes]

Two more data points:

• switching from glibc to jemalloc and the test case passes.
• this seems to only occur when using RocksDB as a dynamic library. Statically linking does not exhibit this behaviour.

[ywave620]

@philmes Could you tell which version of libc and libc++ you are using