Merge remote-tracking branch 'origin/unstable' into memory-commands-d…

…uring-loading

.github/ISSUE_TEMPLATE/config.yml

@@ -9,6 +9,9 @@ contact_links:
   - name: Chat with us on Matrix?
     url: https://matrix.to/#/#valkey:matrix.org
     about: We are on Matrix too!
+  - name: Chat with us on Slack?
+    url: https://join.slack.com/t/valkey-oss-developer/shared_invite/zt-2nxs51chx-EB9hu9Qdch3GMfRcztTSkQ
+    about: We are on Slack too!
   - name: Documentation issue?
     url: https://github.com/valkey-io/valkey-doc/issues
     about: Report it on the valkey-doc repo.

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
       - name: make
         # Fail build if there are warnings
         # build with TLS just for compilation coverage
-        run: make -j4 all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes
+        run: make -j4 all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes USE_FAST_FLOAT=yes
       - name: test
         run: |
           sudo apt-get install tcl8.6 tclx
@@ -108,23 +108,30 @@ jobs:
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: make
-        run: make -j3 all-with-unit-tests SERVER_CFLAGS='-Werror'
+        # Build with additional upcoming features
+        run: make -j3 all-with-unit-tests SERVER_CFLAGS='-Werror' USE_FAST_FLOAT=yes
 
   build-32bit:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: make
+        # Fast float requires C++ 32-bit libraries to compile on 64-bit ubuntu
+        # machine i.e. "-cross" suffixed version. Cross-compiling c++ to 32-bit
+        # also requires multilib support for g++ compiler i.e. "-multilib"
+        # suffixed version of g++. g++-multilib generally includes libstdc++.
+        # *cross version as well, but it is also added explicitly just in case.
         run: |
-          sudo apt-get update && sudo apt-get install libc6-dev-i386
-          make -j4 SERVER_CFLAGS='-Werror' 32bit
+          sudo apt-get update
+          sudo apt-get install libc6-dev-i386 libstdc++-11-dev-i386-cross gcc-multilib g++-multilib
+          make -j4 SERVER_CFLAGS='-Werror' 32bit USE_FAST_FLOAT=yes
 
   build-libc-malloc:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: make
-        run: make -j4 SERVER_CFLAGS='-Werror' MALLOC=libc
+        run: make -j4 SERVER_CFLAGS='-Werror' MALLOC=libc USE_FAST_FLOAT=yes
 
   build-almalinux8-jemalloc:
     runs-on: ubuntu-latest
@@ -134,8 +141,8 @@ jobs:
 
       - name: make
         run: |
-          dnf -y install epel-release gcc make procps-ng which
-          make -j4 SERVER_CFLAGS='-Werror'
+          dnf -y install epel-release gcc gcc-c++ make procps-ng which
+          make -j4 SERVER_CFLAGS='-Werror' USE_FAST_FLOAT=yes
 
   format-yaml:
     runs-on: ubuntu-latest

.github/workflows/daily.yml

@@ -86,7 +86,7 @@ jobs:
         (github.event_name == 'schedule' && github.repository == 'valkey-io/valkey') ||
         (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'run-extra-tests') || github.event.pull_request.base.ref != 'unstable'))) &&
       !contains(github.event.inputs.skipjobs, 'fortify')
-    container: ubuntu:lunar
+    container: ubuntu:plucky
     timeout-minutes: 14400
     steps:
       - name: prep
@@ -319,7 +319,7 @@ jobs:
           ref: ${{ env.GITHUB_HEAD_REF }}
       - name: make
         run: |
-          make BUILD_TLS=yes SERVER_CFLAGS='-Werror'
+          make BUILD_TLS=yes SERVER_CFLAGS='-Werror' USE_FAST_FLOAT=yes
       - name: testprep
         run: |
           sudo apt-get install tcl8.6 tclx tcl-tls
@@ -990,7 +990,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, macos-14]
+        os: [macos-13, macos-14]
     runs-on: ${{ matrix.os }}
     if: |
       (github.event_name == 'workflow_dispatch' ||
@@ -1019,7 +1019,7 @@ jobs:
         run: make SERVER_CFLAGS='-Werror'
 
   test-freebsd:
-    runs-on: macos-12
+    runs-on: macos-13
     if: |
       (github.event_name == 'workflow_dispatch' ||
         (github.event_name == 'schedule' && github.repository == 'valkey-io/valkey') ||

.gitignore

@@ -51,3 +51,5 @@ tests/rdma/rdma-test
 tags
 build-debug/
 build-release/
+cmake-build-debug/
+cmake-build-release/

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.10)
 
 # Must be done first
 if (APPLE)

README.md

@@ -297,19 +297,19 @@ Other options supported by Valkey's `CMake` build system:
 
 ## Special build flags
 
-- `-DBUILD_TLS=<on|off|module>` enable TLS build for Valkey
-- `-DBUILD_RDMA=<off|module>` enable RDMA module build (only module mode supported)
+- `-DBUILD_TLS=<yes|no>` enable TLS build for Valkey. Default: `no`
+- `-DBUILD_RDMA=<no|module>` enable RDMA module build (only module mode supported). Default: `no`
 - `-DBUILD_MALLOC=<libc|jemalloc|tcmalloc|tcmalloc_minimal>` choose the allocator to use. Default on Linux: `jemalloc`, for other OS: `libc`
-- `-DBUILD_SANITIZER=<address|thread|undefined>` build with address sanitizer enabled
-- `-DBUILD_UNIT_TESTS=[1|0]`  when set, the build will produce the executable `valkey-unit-tests`
-- `-DBUILD_TEST_MODULES=[1|0]`  when set, the build will include the modules located under the `tests/modules` folder
-- `-DBUILD_EXAMPLE_MODULES=[1|0]`  when set, the build will include the example modules located under the `src/modules` folder
+- `-DBUILD_SANITIZER=<address|thread|undefined>` build with address sanitizer enabled. Default: disabled (no sanitizer)
+- `-DBUILD_UNIT_TESTS=[yes|no]`  when set, the build will produce the executable `valkey-unit-tests`. Default: `no`
+- `-DBUILD_TEST_MODULES=[yes|no]`  when set, the build will include the modules located under the `tests/modules` folder. Default: `no`
+- `-DBUILD_EXAMPLE_MODULES=[yes|no]`  when set, the build will include the example modules located under the `src/modules` folder. Default: `no`
 
 ## Common flags
 
 - `-DCMAKE_BUILD_TYPE=<Debug|Release...>` define the build type, see CMake manual for more details
 - `-DCMAKE_INSTALL_PREFIX=/installation/path` override this value to define a custom install prefix. Default: `/usr/local`
-- `-G<Generator Name>` generate build files for "Generator Name". By default, CMake will generate `Makefile`s.
+- `-G"<Generator Name>"` generate build files for "Generator Name". By default, CMake will generate `Makefile`s.
 
 ## Verbose build
 

cmake/Modules/SourceFiles.cmake

@@ -74,6 +74,7 @@ set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/geohash.c
     ${CMAKE_SOURCE_DIR}/src/geohash_helper.c
     ${CMAKE_SOURCE_DIR}/src/childinfo.c
+    ${CMAKE_SOURCE_DIR}/src/allocator_defrag.c
     ${CMAKE_SOURCE_DIR}/src/defrag.c
     ${CMAKE_SOURCE_DIR}/src/siphash.c
     ${CMAKE_SOURCE_DIR}/src/rax.c

cmake/Modules/Utils.cmake

@@ -100,3 +100,16 @@ function (valkey_parse_build_option OPTION_VALUE OUT_ARG_ENUM)
             PARENT_SCOPE)
     endif ()
 endfunction ()
+
+function (valkey_pkg_config PKGNAME OUT_VARIABLE)
+    if (NOT FOUND_PKGCONFIG)
+        # Locate pkg-config once
+        find_package(PkgConfig REQUIRED)
+        set(FOUND_PKGCONFIG 1)
+    endif ()
+    pkg_check_modules(__PREFIX REQUIRED ${PKGNAME})
+    message(STATUS "Found library for '${PKGNAME}': ${__PREFIX_LIBRARIES}")
+    set(${OUT_VARIABLE}
+        "${__PREFIX_LIBRARIES}"
+        PARENT_SCOPE)
+endfunction ()

cmake/Modules/ValkeySetup.cmake

@@ -74,9 +74,11 @@ endmacro ()
 macro (valkey_build_and_install_bin target sources ld_flags libs link_name)
     add_executable(${target} ${sources})
 
-    if (USE_JEMALLOC)
-        # Using jemalloc
-        target_link_libraries(${target} jemalloc)
+    if (USE_JEMALLOC
+        OR USE_TCMALLOC
+        OR USE_TCMALLOC_MINIMAL)
+        # Using custom allocator
+        target_link_libraries(${target} ${ALLOCATOR_LIB})
     endif ()
 
     # Place this line last to ensure that ${ld_flags} is placed last on the linker line
@@ -151,16 +153,23 @@ endif ()
 if (BUILD_MALLOC)
     if ("${BUILD_MALLOC}" STREQUAL "jemalloc")
         set(MALLOC_LIB "jemalloc")
+        set(ALLOCATOR_LIB "jemalloc")
         add_valkey_server_compiler_options("-DUSE_JEMALLOC")
         set(USE_JEMALLOC 1)
     elseif ("${BUILD_MALLOC}" STREQUAL "libc")
         set(MALLOC_LIB "libc")
     elseif ("${BUILD_MALLOC}" STREQUAL "tcmalloc")
         set(MALLOC_LIB "tcmalloc")
+        valkey_pkg_config(libtcmalloc ALLOCATOR_LIB)
+
         add_valkey_server_compiler_options("-DUSE_TCMALLOC")
+        set(USE_TCMALLOC 1)
     elseif ("${BUILD_MALLOC}" STREQUAL "tcmalloc_minimal")
         set(MALLOC_LIB "tcmalloc_minimal")
+        valkey_pkg_config(libtcmalloc_minimal ALLOCATOR_LIB)
+
         add_valkey_server_compiler_options("-DUSE_TCMALLOC")
+        set(USE_TCMALLOC_MINIMAL 1)
     else ()
         message(FATAL_ERROR "BUILD_MALLOC can be one of: jemalloc, libc, tcmalloc or tcmalloc_minimal")
     endif ()
@@ -202,16 +211,12 @@ if (BUILD_RDMA)
         if (USE_RDMA EQUAL 2) # Module
             message(STATUS "Building RDMA as module")
             add_valkey_server_compiler_options("-DUSE_RDMA=2")
-            find_package(PkgConfig REQUIRED)
 
             # Locate librdmacm & libibverbs, fail if we can't find them
-            pkg_check_modules(RDMACM REQUIRED librdmacm)
-            pkg_check_modules(IBVERBS REQUIRED libibverbs)
+            valkey_pkg_config(librdmacm RDMACM_LIBS)
+            valkey_pkg_config(libibverbs IBVERBS_LIBS)
 
-            message(STATUS "${RDMACM_LINK_LIBRARIES};${IBVERBS_LINK_LIBRARIES}")
-            list(APPEND RDMA_LIBS "${RDMACM_LIBRARIES};${IBVERBS_LIBRARIES}")
-            unset(RDMACM_LINK_LIBRARIES CACHE)
-            unset(IBVERBS_LINK_LIBRARIES CACHE)
+            list(APPEND RDMA_LIBS "${RDMACM_LIBS};${IBVERBS_LIBS}")
             set(BUILD_RDMA_MODULE 1)
         elseif (USE_RDMA EQUAL 1)
             # RDMA can only be built as a module. So disable it
@@ -266,17 +271,18 @@ endif ()
 
 # Sanitizer
 if (BUILD_SANITIZER)
-    # For best results, force libc
-    set(MALLOC_LIB, "libc")
+    # Common CFLAGS
+    list(APPEND VALKEY_SANITAIZER_CFLAGS "-fno-sanitize-recover=all")
+    list(APPEND VALKEY_SANITAIZER_CFLAGS "-fno-omit-frame-pointer")
     if ("${BUILD_SANITIZER}" STREQUAL "address")
-        add_valkey_server_compiler_options("-fsanitize=address -fno-sanitize-recover=all -fno-omit-frame-pointer")
-        add_valkey_server_linker_option("-fsanitize=address")
+        list(APPEND VALKEY_SANITAIZER_CFLAGS "-fsanitize=address")
+        list(APPEND VALKEY_SANITAIZER_LDFLAGS "-fsanitize=address")
     elseif ("${BUILD_SANITIZER}" STREQUAL "thread")
-        add_valkey_server_compiler_options("-fsanitize=thread -fno-sanitize-recover=all -fno-omit-frame-pointer")
-        add_valkey_server_linker_option("-fsanitize=thread")
+        list(APPEND VALKEY_SANITAIZER_CFLAGS "-fsanitize=thread")
+        list(APPEND VALKEY_SANITAIZER_LDFLAGS "-fsanitize=thread")
     elseif ("${BUILD_SANITIZER}" STREQUAL "undefined")
-        add_valkey_server_compiler_options("-fsanitize=undefined -fno-sanitize-recover=all -fno-omit-frame-pointer")
-        add_valkey_server_linker_option("-fsanitize=undefined")
+        list(APPEND VALKEY_SANITAIZER_CFLAGS "-fsanitize=undefined")
+        list(APPEND VALKEY_SANITAIZER_LDFLAGS "-fsanitize=undefined")
     else ()
         message(FATAL_ERROR "Unknown sanitizer: ${BUILD_SANITIZER}")
     endif ()
@@ -366,7 +372,6 @@ include(SourceFiles)
 
 # Clear the below variables from the cache
 unset(CMAKE_C_FLAGS CACHE)
-unset(BUILD_SANITIZER CACHE)
 unset(VALKEY_SERVER_LDFLAGS CACHE)
 unset(VALKEY_SERVER_CFLAGS CACHE)
 unset(PYTHON_EXE CACHE)

deps/Makefile

@@ -42,6 +42,7 @@ distclean:
 	-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
 	-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
 	-(cd fpconv && $(MAKE) clean) > /dev/null || true
+	-(cd fast_float_c_interface && $(MAKE) clean) > /dev/null || true
 	-(rm -f .make-*)
 
 .PHONY: distclean
@@ -116,3 +117,9 @@ jemalloc: .make-prerequisites
 	cd jemalloc && $(MAKE) lib/libjemalloc.a
 
 .PHONY: jemalloc
+
+fast_float_c_interface: .make-prerequisites
+	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+	cd fast_float_c_interface && $(MAKE)
+
+.PHONY: fast_float_c_interface

deps/README.md

@@ -6,6 +6,7 @@ should be provided by the operating system.
 * **linenoise** is a readline replacement. It is developed by the same authors of Valkey but is managed as a separated project and updated as needed.
 * **lua** is Lua 5.1 with minor changes for security and additional libraries.
 * **hdr_histogram** Used for per-command latency tracking histograms.
+* **fast_float** is a replacement for strtod to convert strings to floats efficiently. 
 
 How to upgrade the above dependencies
 ===
@@ -105,3 +106,17 @@ We use a customized version based on master branch commit e4448cf6d1cd08fff51981
 2. Copy updated files from newer version onto files in /hdr_histogram.
 3. Apply the changes from 1 above to the updated files.
 
+fast_float
+---
+The fast_float library provides fast header-only implementations for the C++ from_chars functions for `float` and `double` types as well as integer types.  These functions convert ASCII strings representing decimal values (e.g., `1.3e10`) into binary types. The functions are much faster than comparable number-parsing functions from existing C++ standard libraries.
+
+Specifically, `fast_float` provides the following function to parse floating-point numbers with a C++17-like syntax (the library itself only requires C++11):
+
+    template <typename T, typename UC = char, typename = FASTFLOAT_ENABLE_IF(is_supported_float_type<T>())>
+    from_chars_result_t<UC> from_chars(UC const *first, UC const *last, T &value, chars_format fmt = chars_format::general);
+
+To upgrade the library,
+1. Check out https://github.com/fastfloat/fast_float/tree/main
+2. cd fast_float
+3. Invoke "python3 ./script/amalgamate.py --output fast_float.h"
+4. Copy fast_float.h file to "deps/fast_float/".