Skip to content

Commit

Permalink
add fixed length str2int / asin str2int memory-mappable hashmap (amzn…
Browse files Browse the repository at this point in the history 
…#295)

add fixed length str2int / asin str2int memory-mappable hashmap
  • Loading branch information
@vpung
vpung authored Aug 29, 2024
1 parent 455db26 commit 37028ca
Show file tree
Hide file tree
Showing 6 changed files with 584 additions and 6 deletions.
2 changes: 1 addition & 1 deletion .github/style_type_check_cfg/.flake8
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[flake8]
ignore = E203,E501,W605,F541
extend-ignore = E203,E501,W605,F541
max_line_length = 100
18 changes: 17 additions & 1 deletion pecos/core/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2070,12 +2070,20 @@ def link_mmap_hashmap_methods(self):
Specify C-lib's Memory-mappable Hashmap methods arguments and return types.
"""
fn_prefix = "mmap_hashmap"
map_type_list = ["str2int", "int2int"]
map_type_list = ["str2int", "fixed_len_str2int", "fixed_len_10_str2int", "int2int"]
key_args_dict = {
"str2int": [
c_char_p, # pointer of key string
c_uint32, # length of key string
],
"fixed_len_str2int": [
c_char_p, # pointer of key string
c_uint32, # length of key string
],
"fixed_len_10_str2int": [
c_char_p, # pointer of key string
c_uint32, # length of key string
],
"int2int": [
c_uint64, # key int64
],
Expand All @@ -2085,6 +2093,14 @@ def link_mmap_hashmap_methods(self):
c_void_p, # List of pointer of key string
POINTER(c_uint32), # List of length of key string
],
"fixed_len_str2int": [
c_void_p, # List of pointer of key string
POINTER(c_uint32), # List of length of key string
],
"fixed_len_10_str2int": [
c_void_p, # List of pointer of key string
POINTER(c_uint32), # List of length of key string
],
"int2int": [
POINTER(c_uint64), # List of key int64
],
Expand Down
24 changes: 23 additions & 1 deletion pecos/core/libpecos.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -661,28 +661,36 @@ extern "C" {

// ==== C Interface of Memory-mappable Hashmap ====

typedef pecos::mmap_hashmap::Str2IntMap mmap_hashmap_str2int;
typedef pecos::mmap_hashmap::Str2IntMap<pecos::mmap_hashmap::details_::AnkerlStr2IntMmapableVector> mmap_hashmap_str2int;
typedef pecos::mmap_hashmap::Str2IntMap<pecos::mmap_hashmap::details_::AnkerlFixedLenStr2IntMmapableVector> mmap_hashmap_fixed_len_str2int;
typedef pecos::mmap_hashmap::Str2IntMap<pecos::mmap_hashmap::details_::AnkerlFixedLen10Str2IntMmapableVector> mmap_hashmap_fixed_len_10_str2int;
typedef pecos::mmap_hashmap::Int2IntMap mmap_hashmap_int2int;

// New
#define MMAP_MAP_NEW(SUFFIX) \
void* mmap_hashmap_new_ ## SUFFIX () { \
return static_cast<void*>(new mmap_hashmap_ ## SUFFIX()); }
MMAP_MAP_NEW(str2int)
MMAP_MAP_NEW(fixed_len_str2int)
MMAP_MAP_NEW(fixed_len_10_str2int)
MMAP_MAP_NEW(int2int)

// Destruct
#define MMAP_MAP_DESTRUCT(SUFFIX) \
void mmap_hashmap_destruct_ ## SUFFIX (void* map_ptr) { \
delete static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr); }
MMAP_MAP_DESTRUCT(str2int)
MMAP_MAP_DESTRUCT(fixed_len_str2int)
MMAP_MAP_DESTRUCT(fixed_len_10_str2int)
MMAP_MAP_DESTRUCT(int2int)

// Save
#define MMAP_MAP_SAVE(SUFFIX) \
void mmap_hashmap_save_ ## SUFFIX (void* map_ptr, const char* map_dir) { \
static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->save(map_dir); }
MMAP_MAP_SAVE(str2int)
MMAP_MAP_SAVE(fixed_len_str2int)
MMAP_MAP_SAVE(fixed_len_10_str2int)
MMAP_MAP_SAVE(int2int)

// Load
Expand All @@ -692,13 +700,17 @@ extern "C" {
map_ptr->load(map_dir, lazy_load); \
return static_cast<void *>(map_ptr); }
MMAP_MAP_LOAD(str2int)
MMAP_MAP_LOAD(fixed_len_str2int)
MMAP_MAP_LOAD(fixed_len_10_str2int)
MMAP_MAP_LOAD(int2int)

// Size
#define MMAP_MAP_SIZE(SUFFIX) \
size_t mmap_hashmap_size_ ## SUFFIX (void* map_ptr) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->size(); }
MMAP_MAP_SIZE(str2int)
MMAP_MAP_SIZE(fixed_len_str2int)
MMAP_MAP_SIZE(fixed_len_10_str2int)
MMAP_MAP_SIZE(int2int)

// Insert
Expand All @@ -707,32 +719,42 @@ extern "C" {
void mmap_hashmap_insert_ ## SUFFIX (void* map_ptr, KEY, uint64_t val) { \
static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->insert(FUNC_CALL_KEY, val); }
MMAP_MAP_INSERT(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_INSERT(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_INSERT(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_INSERT(int2int, uint64_t key, key)

// Get
#define MMAP_MAP_GET(SUFFIX, KEY, FUNC_CALL_KEY) \
uint64_t mmap_hashmap_get_ ## SUFFIX (void* map_ptr, KEY) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->get(FUNC_CALL_KEY); }
MMAP_MAP_GET(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET(int2int, uint64_t key, key)

#define MMAP_MAP_GET_W_DEFAULT(SUFFIX, KEY, FUNC_CALL_KEY) \
uint64_t mmap_hashmap_get_w_default_ ## SUFFIX (void* map_ptr, KEY, uint64_t def_val) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->get_w_default(FUNC_CALL_KEY, def_val); }
MMAP_MAP_GET_W_DEFAULT(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET_W_DEFAULT(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET_W_DEFAULT(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET_W_DEFAULT(int2int, uint64_t key, key)

#define MMAP_MAP_BATCH_GET_W_DEFAULT(SUFFIX, KEY, FUNC_CALL_KEY) \
void mmap_hashmap_batch_get_w_default_ ## SUFFIX (void* map_ptr, const uint32_t n_key, KEY, uint64_t def_val, uint64_t* vals, const int threads) { \
static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->batch_get_w_default(n_key, FUNC_CALL_KEY, def_val, vals, threads); }
MMAP_MAP_BATCH_GET_W_DEFAULT(str2int, KEY_SINGLE_ARG(const char* const* keys, const uint32_t* keys_lens), KEY_SINGLE_ARG(keys, keys_lens))
MMAP_MAP_BATCH_GET_W_DEFAULT(fixed_len_str2int, KEY_SINGLE_ARG(const char* const* keys, const uint32_t* keys_lens), KEY_SINGLE_ARG(keys, keys_lens))
MMAP_MAP_BATCH_GET_W_DEFAULT(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* const* keys, const uint32_t* keys_lens), KEY_SINGLE_ARG(keys, keys_lens))
MMAP_MAP_BATCH_GET_W_DEFAULT(int2int, const uint64_t* key, key)

// Contains
#define MMAP_MAP_CONTAINS(SUFFIX, KEY, FUNC_CALL_KEY) \
bool mmap_hashmap_contains_ ## SUFFIX (void* map_ptr, KEY) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->contains(FUNC_CALL_KEY); }
MMAP_MAP_CONTAINS(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_CONTAINS(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_CONTAINS(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_CONTAINS(int2int, uint64_t key, key)


Expand Down
Loading

0 comments on commit 37028ca

Please sign in to comment.