Support RDMA for valkey-cli and benchmark (#2059)

Add '--rdma' for both valkey-cli and valkey-benchmark to enable RDMA.

Valkey has already replaced dependency `hiredis` with `libvalkey`
(#2032), and libvalkey also supports RDMA.

---------

Signed-off-by: zhenwei pi <zhenwei.pi@linux.dev>
Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>

Author: pizhenwei

cmake/Modules/ValkeySetup.cmake

@@ -89,6 +89,15 @@ macro (valkey_build_and_install_bin target sources ld_flags libs link_name)
         target_link_libraries(${target} OpenSSL::SSL valkey::valkey_tls)
     endif ()
 
+    if (USE_RDMA)
+        # Add required libraries needed for RDMA
+        # Bug in libvalkey 0.1.0: The order is important and we need to link
+        # valkey::valkey after valkey::valkey_rdma. This will be fixed upstream.
+        # TODO: Next time we lift libvalkey, remove valkey::valkey from
+        # the next line.
+        target_link_libraries(${target} valkey::valkey_rdma valkey::valkey)
+    endif ()
+
     if (IS_FREEBSD)
         target_link_libraries(${target} execinfo)
     endif ()
@@ -101,30 +110,6 @@ macro (valkey_build_and_install_bin target sources ld_flags libs link_name)
     valkey_create_symlink(${target} ${link_name})
 endmacro ()
 
-# Helper function that defines, builds and installs `target` module.
-macro (valkey_build_and_install_module target sources ld_flags libs)
-    add_library(${target} SHARED ${sources})
-
-    if (USE_JEMALLOC)
-        # Using jemalloc
-        target_link_libraries(${target} jemalloc)
-    endif ()
-
-    # Place this line last to ensure that ${ld_flags} is placed last on the linker line
-    target_link_libraries(${target} ${libs} ${ld_flags})
-    if (USE_TLS)
-        # Add required libraries needed for TLS
-        target_link_libraries(${target} OpenSSL::SSL valkey::valkey_tls)
-    endif ()
-
-    if (IS_FREEBSD)
-        target_link_libraries(${target} execinfo)
-    endif ()
-
-    # Install cli tool and create a redis symbolic link
-    valkey_install_bin(${target})
-endmacro ()
-
 # Determine if we are building in Release or Debug mode
 if (CMAKE_BUILD_TYPE MATCHES Debug OR CMAKE_BUILD_TYPE MATCHES DebugFull)
     set(VALKEY_DEBUG_BUILD 1)

deps/CMakeLists.txt

@@ -14,7 +14,13 @@ if (USE_TLS) # Module or no module
     message(STATUS "Building valkey_tls")
     set(ENABLE_TLS
         ON
-        CACHE BOOL "Should we test SSL connections")
+        CACHE BOOL "If TLS support should be compiled or not")
+endif ()
+if (USE_RDMA) # Module or no module
+    message(STATUS "Building valkey_rdma")
+    set(ENABLE_RDMA
+        ON
+        CACHE BOOL "If RDMA support should be compiled or not")
 endif ()
 # Let libvalkey use sds and dict provided by valkey.
 set(DICT_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/src)
@@ -29,3 +35,4 @@ add_subdirectory(hdr_histogram)
 unset(BUILD_SHARED_LIBS CACHE)
 unset(DISABLE_TESTS CACHE)
 unset(ENABLE_TLS CACHE)
+unset(ENABLE_RDMA CACHE)

deps/Makefile

@@ -52,6 +52,10 @@ ifneq (,$(filter $(BUILD_TLS),yes module))
     LIBVALKEY_MAKE_FLAGS += USE_TLS=1
 endif
 
+ifneq (,$(filter $(BUILD_RDMA),yes module))
+    LIBVALKEY_MAKE_FLAGS += USE_RDMA=1
+endif
+
 libvalkey: .make-prerequisites
 	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
 	cd libvalkey && $(MAKE) static $(LIBVALKEY_MAKE_FLAGS)

src/Makefile

@@ -340,14 +340,15 @@ endif
 
 ifeq ($(BUILD_RDMA),yes)
 	FINAL_CFLAGS+=-DUSE_RDMA=$(BUILD_YES) -DBUILD_RDMA_MODULE=$(BUILD_NO)
-	FINAL_LIBS += $(RDMA_LIBS)
+	FINAL_LIBS += ../deps/libvalkey/lib/libvalkey_rdma.a $(RDMA_LIBS)
 endif
 
 RDMA_MODULE=
 RDMA_MODULE_NAME:=valkey-rdma$(PROG_SUFFIX).so
 RDMA_MODULE_CFLAGS:=$(FINAL_CFLAGS)
 ifeq ($(BUILD_RDMA),module)
 	FINAL_CFLAGS+=-DUSE_RDMA=$(BUILD_MODULE)
+	RDMA_CLIENT_LIBS = ../deps/libvalkey/lib/libvalkey_rdma.a $(RDMA_LIBS)
 	RDMA_MODULE=$(RDMA_MODULE_NAME)
 	RDMA_MODULE_CFLAGS+=-DUSE_RDMA=$(BUILD_MODULE) -DBUILD_RDMA_MODULE=$(BUILD_MODULE) $(RDMA_LIBS)
 endif
@@ -522,11 +523,11 @@ $(RDMA_MODULE_NAME): $(SERVER_NAME)
 
 # valkey-cli
 $(ENGINE_CLI_NAME): $(ENGINE_CLI_OBJ)
-	$(SERVER_LD) -o $@ $^ ../deps/libvalkey/lib/libvalkey.a ../deps/linenoise/linenoise.o ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS)
+	$(SERVER_LD) -o $@ $^ ../deps/libvalkey/lib/libvalkey.a ../deps/linenoise/linenoise.o ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS) $(RDMA_CLIENT_LIBS)
 
 # valkey-benchmark
 $(ENGINE_BENCHMARK_NAME): $(ENGINE_BENCHMARK_OBJ)
-	$(SERVER_LD) -o $@ $^ ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS)
+	$(SERVER_LD) -o $@ $^ ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS) $(RDMA_CLIENT_LIBS)
 
 DEP = $(ENGINE_SERVER_OBJ:%.o=%.d) $(ENGINE_CLI_OBJ:%.o=%.d) $(ENGINE_BENCHMARK_OBJ:%.o=%.d)
 -include $(DEP)

src/cli_common.c

@@ -32,11 +32,11 @@
 #include "cli_common.h"
 #include "version.h"
 
+#include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include <errno.h>
-#include <valkey/valkey.h>
 #include "sds.h"
 #include <unistd.h>
 #include <string.h>
@@ -46,6 +46,9 @@
 #include <openssl/err.h>
 #include <valkey/tls.h>
 #endif
+#ifdef USE_RDMA
+#include <valkey/rdma.h>
+#endif
 
 #define UNUSED(V) ((void)V)
 
@@ -427,25 +430,29 @@ sds cliVersion(void) {
 }
 
 /* This is a wrapper to call valkeyConnect or valkeyConnectWithTimeout. */
-valkeyContext *valkeyConnectWrapper(const char *ip, int port, const struct timeval tv, int nonblock) {
+valkeyContext *valkeyConnectWrapper(enum valkeyConnectionType ct, const char *ip_or_path, int port, const struct timeval tv, int nonblock) {
     valkeyOptions options = {0};
-    VALKEY_OPTIONS_SET_TCP(&options, ip, port);
 
-    if (tv.tv_sec || tv.tv_usec) {
-        options.connect_timeout = &tv;
-    }
+    switch (ct) {
+    case VALKEY_CONN_TCP:
+        VALKEY_OPTIONS_SET_TCP(&options, ip_or_path, port);
+        break;
 
-    if (nonblock) {
-        options.options |= VALKEY_OPT_NONBLOCK;
-    }
+    case VALKEY_CONN_UNIX:
+        VALKEY_OPTIONS_SET_UNIX(&options, ip_or_path);
+        break;
 
-    return valkeyConnectWithOptions(&options);
-}
+    case VALKEY_CONN_RDMA:
+#ifdef USE_RDMA
+        VALKEY_OPTIONS_SET_RDMA(&options, ip_or_path, port);
+        break;
+#else
+        assert(0); /* requesting RDMA connection without RDMA support??? */
+#endif
 
-/* This is a wrapper to call valkeyConnectUnix or valkeyConnectUnixWithTimeout. */
-valkeyContext *valkeyConnectUnixWrapper(const char *path, const struct timeval tv, int nonblock) {
-    valkeyOptions options = {0};
-    VALKEY_OPTIONS_SET_UNIX(&options, path);
+    default:
+        assert(0); /* this should not happen */
+    }
 
     if (tv.tv_sec || tv.tv_usec) {
         options.connect_timeout = &tv;

src/cli_common.h

@@ -53,7 +53,6 @@ sds escapeJsonString(sds s, const char *p, size_t len);
 
 sds cliVersion(void);
 
-valkeyContext *valkeyConnectWrapper(const char *ip, int port, const struct timeval tv, int nonblock);
-valkeyContext *valkeyConnectUnixWrapper(const char *path, const struct timeval tv, int nonblock);
+valkeyContext *valkeyConnectWrapper(enum valkeyConnectionType ct, const char *ip_or_path, int port, const struct timeval tv, int nonblock);
 
 #endif /* __CLICOMMON_H */