Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for compiling with mimalloc #363

Open
wants to merge 29 commits into
base: unstable
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
8f202d2
Rename remaining test procedures (#355)
Shivshankar-Reddy Apr 24, 2024
4bdc923
Add support for compiling with mimalloc
Apr 24, 2024
c5d5199
fix
Apr 24, 2024
0d7c60a
Add support to use system libraries
Apr 24, 2024
743d978
Rename redis.tcl to valkey.tcl (#283)
hwware Apr 24, 2024
2778f3b
Update COPYING to Valkey (#32)
PatrickJS Apr 24, 2024
9fc71a3
Update redis* to valkey* in syscheck.c (#365)
hwware Apr 25, 2024
08684da
Update redis* to valkey* in server.c and module.c (#367)
artikell Apr 25, 2024
f144dc0
Rename redis to valkey in test suite logs and test names. (#366)
Shivshankar-Reddy Apr 25, 2024
345e7a6
Detect accept4() on specific versions of various platforms (#294)
panjf2000 Apr 25, 2024
9f93823
Remove the option for vendored mimalloc
Apr 26, 2024
c4b890a
remove mimalloc submodule
Apr 26, 2024
1f1ca13
Merge branch 'valkey-io:unstable' into mimalloc-support
WM0323 Apr 26, 2024
4a39d8c
remove changes in the deps/Makefile
Apr 26, 2024
658aec3
remove USE_SYSTEM_MIMALLOC option
Apr 26, 2024
f16f92f
Explicitly override malloc/free etc when using mimalloc
Apr 26, 2024
32d07dd
update the README.md with mimalloc install instructions and zmalloc.h
Apr 29, 2024
fcc724b
Merge branch 'valkey-io:unstable' into mimalloc-support
WM0323 Apr 29, 2024
1219f1c
remove unused expansion
Apr 29, 2024
0438848
Merge branch 'mimalloc-support' of https://github.com/WM0323/valkey i…
Apr 29, 2024
65ef92f
add defrag_supported field to INFO command
Apr 29, 2024
371641d
fix
Apr 29, 2024
b1431fa
remove defrag_supported macro
Apr 30, 2024
2f00910
fix
Apr 30, 2024
c9c3733
fix2
Apr 30, 2024
a594b55
fix
Apr 30, 2024
2466cb4
fix3
Apr 30, 2024
1854578
add tcl test for defrag_supported
May 2, 2024
e391645
Merge branch 'valkey-io:unstable' into mimalloc-support
WM0323 May 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,25 @@ If you want to see a more verbose output, use the following:

% make V=1

Using mimalloc with Valkey
-------------

[mimalloc](https://github.com/microsoft/mimalloc) is an advanced memory allocator that can improve the performance of applications. You can optionally build Valkey using mimalloc by following these steps:

Before building Valkey with mimalloc, you need to install mimalloc on your system.

% git clone https://github.com/microsoft/mimalloc.git
% mkdir -p mimalloc/out/release
% cd mimalloc/out/release
% cmake ../.. -DMI_INSTALL_TOPLEVEL=ON
% sudo make install

This will install mimalloc libraries and headers in /usr/local/lib and /usr/local/include.

Build Valkey with mimalloc:

% make MALLOC=mimalloc

Running Valkey
-------------

Expand Down
11 changes: 11 additions & 0 deletions src/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ ifeq ($(USE_JEMALLOC),no)
MALLOC=libc
endif

ifeq ($(USE_MIMALLOC),yes)
MALLOC=mimalloc
endif

ifdef SANITIZER
ifeq ($(SANITIZER),address)
MALLOC=libc
Expand Down Expand Up @@ -285,6 +289,13 @@ ifeq ($(MALLOC),jemalloc)
FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS)
endif

ifeq ($(MALLOC),mimalloc)
FINAL_CFLAGS+= -DUSE_MIMALLOC -I/usr/local/include
FINAL_LDFLAGS+= -L/usr/local/lib
FINAL_LIBS := -lmimalloc $(FINAL_LIBS)
endif


# LIBSSL & LIBCRYPTO
LIBSSL_LIBS=
LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?)
Expand Down
3 changes: 2 additions & 1 deletion src/server.c
Original file line number Diff line number Diff line change
Expand Up @@ -5757,7 +5757,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
"mem_overhead_db_hashtable_rehashing:%zu\r\n", mh->overhead_db_hashtable_rehashing,
"active_defrag_running:%d\r\n", server.active_defrag_running,
"lazyfree_pending_objects:%zu\r\n", lazyfreeGetPendingObjectsCount(),
"lazyfreed_objects:%zu\r\n", lazyfreeGetFreedObjectsCount()));
"lazyfreed_objects:%zu\r\n", lazyfreeGetFreedObjectsCount(),
"defrag_supported: %s\r\n", defrag_supported));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe add a tcl test to verify the behavior.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated in other.tcl file.

freeMemoryOverheadData(mh);
}

Expand Down
6 changes: 6 additions & 0 deletions src/zmalloc.c
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ void zlibc_free(void *ptr) {
#define free(ptr) je_free(ptr)
#define mallocx(size,flags) je_mallocx(size,flags)
#define dallocx(ptr,flags) je_dallocx(ptr,flags)
/* Explicitly override malloc/free etc when using mimalloc. */
#elif defined(USE_MIMALLOC)
#define malloc(size) mi_malloc(size)
#define calloc(count,size) mi_calloc(count,size)
#define realloc(ptr,size) mi_realloc(ptr,size)
#define free(ptr) mi_free(ptr)
#endif

#define update_zmalloc_stat_alloc(__n) atomicIncr(used_memory,(__n))
Expand Down
20 changes: 20 additions & 0 deletions src/zmalloc.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@

#if defined(USE_TCMALLOC)
#define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR))
#define defrag_supported "no"
#include <google/tcmalloc.h>
#if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1)
#define HAVE_MALLOC_SIZE 1
Expand All @@ -47,6 +48,7 @@

#elif defined(USE_JEMALLOC)
#define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX))
#define defrag_supported "yes"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would rather simply compare the allocator, if it's JEMALLOC print yes else no for now. Won't require us to add another macro.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, good suggestion:) I will update in server.c file: directly check which memory allocator is being used, avoids introducing new macros.

#include <jemalloc/jemalloc.h>
#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2)
#define HAVE_MALLOC_SIZE 1
Expand All @@ -55,6 +57,20 @@
#error "Newer version of jemalloc required"
#endif

#elif defined(USE_MIMALLOC)
#define defrag_supported "no"
#include <mimalloc.h>
#define ZMALLOC_LIB ("mimalloc-" __xstr(MI_MALLOC_VERSION))
#define MI_VERSION_MAJOR (MI_MALLOC_VERSION / 100)
#define MI_VERSION_MINOR ((MI_MALLOC_VERSION / 10) % 10)
#define MI_VERSION_PATCH (MI_MALLOC_VERSION % 10)
#if (MI_VERSION_MAJOR == 1 && MI_VERSION_MINOR >= 8) || (MI_VERSION_PATCH > 1)
#define HAVE_MALLOC_SIZE 1
#define zmalloc_size(p) mi_usable_size(p)
#else
#error "Newer version of mimalloc required"
#endif

#elif defined(__APPLE__)
#include <malloc/malloc.h>
#define HAVE_MALLOC_SIZE 1
Expand All @@ -73,6 +89,10 @@
#define ZMALLOC_LIB "libc"
#define USE_LIBC 1

#ifndef defrag_supported
#define defrag_supported "no"
#endif

#if !defined(NO_MALLOC_USABLE_SIZE) && \
(defined(__GLIBC__) || defined(__FreeBSD__) || \
defined(__DragonFly__) || defined(__HAIKU__) || \
Expand Down
Loading