Add nvtext substring duplication APIs (Part 1) (#18585)

Adds new set of nvtext substring deduplication APIs

```cpp
std::unique_ptr<cudf::column> nvtext::substring_duplicates(
  cudf::strings_column_view const& input,
  cudf::size_type min_width,
  rmm::cuda_stream_view stream,
  rmm::device_async_resource_ref mr);
```
This finds and returns any duplicate substrings of at least `min_width` bytes within the `input` column.

```cpp
std::unique_ptr<rmm::device_uvector<int32_t>> build_suffix_array(
  cudf::strings_column_view const& input,
  int32_t min_width,
  rmm::cuda_stream_view stream,
  rmm::device_async_resource_ref mr);
```
Just builds the suffix array.

The `build_suffix_array` returns int32 values per input byte. So the output is 4x the input size.
Additionally, the internal sort routine requires an extra 4x of temporary memory.
This means the memory footprint for `build_suffix_array` will be about 8x the size of the input.
Also, this function will be about 95% of the runtime and care should be taken to not to call this repeatedly on the same input data.

Authors:
  - David Wendt (https://github.com/davidwendt)

Approvers:
  - Matthew Roeschke (https://github.com/mroeschke)
  - Shruti Shivakumar (https://github.com/shrshi)
  - Tianyu Liu (https://github.com/kingcrimsontianyu)
  - Bradley Dice (https://github.com/bdice)

URL: #18585

Author: davidwendt

cpp/CMakeLists.txt

@@ -29,6 +29,7 @@ project(
   VERSION "${RAPIDS_VERSION}"
   LANGUAGES C CXX CUDA
 )
+
 if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.5)
   message(
     FATAL_ERROR
@@ -45,7 +46,6 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 
 # ##################################################################################################
 # * build options ---------------------------------------------------------------------------------
-
 option(USE_NVTX "Build with NVTX support" ON)
 option(BUILD_TESTS "Configure CMake to build tests" ON)
 option(BUILD_BENCHMARKS "Configure CMake to build (google & nvbench) benchmarks" OFF)
@@ -63,6 +63,7 @@ option(
          stream to external libraries."
   OFF
 )
+
 # Option to add all symbols to the dynamic symbol table in the library file, allowing to retrieve
 # human-readable stacktrace for debugging.
 option(
@@ -71,19 +72,23 @@ option(
   OFF
 )
 option(DISABLE_DEPRECATION_WARNINGS "Disable warnings generated from deprecated declarations." OFF)
+
 # Option to enable line info in CUDA device compilation to allow introspection when profiling /
 # memchecking
 option(CUDA_ENABLE_LINEINFO
        "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF
 )
 option(CUDA_WARNINGS_AS_ERRORS "Enable -Werror=all-warnings for all CUDA compilation" ON)
+
 # cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking
 option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)
 
 set(DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL ON)
+
 if(CUDA_STATIC_RUNTIME OR NOT BUILD_SHARED_LIBS)
   set(DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL OFF)
 endif()
+
 option(
   CUDF_BUILD_STREAMS_TEST_UTIL
   "Whether to build the utilities for stream testing contained in libcudf"
@@ -127,6 +132,7 @@ message(VERBOSE
 rapids_cmake_build_type("Release")
 set(CUDF_BUILD_TESTS ${BUILD_TESTS})
 set(CUDF_BUILD_BENCHMARKS ${BUILD_BENCHMARKS})
+
 if(BUILD_TESTS AND NOT CUDF_BUILD_TESTUTIL)
   message(
     FATAL_ERROR
@@ -175,12 +181,14 @@ if(CUDF_CLANG_TIDY)
   string(REGEX MATCH "LLVM version ([0-9]+\\.[0-9]+)\\.[0-9]+" LLVM_VERSION_MATCH
                "${CLANG_TIDY_OUTPUT}"
   )
+
   # Discard the patch version and allow it to float. Empirically, results between patch versions are
   # mostly stable, and looking at available packages on some package managers sometimes patch
   # versions are skipped so we don't want to constrain to a patch version that the user can't
   # install.
   set(LLVM_VERSION "${CMAKE_MATCH_1}")
   set(expected_clang_tidy_version 19.1)
+
   if(NOT expected_clang_tidy_version VERSION_EQUAL LLVM_VERSION)
     message(
       FATAL_ERROR
@@ -216,6 +224,7 @@ function(enable_static_checkers target)
       )
     endif()
   endif()
+
   if(_LINT_IWYU)
     # A few extra warnings pop up when building with IWYU. I'm not sure why, but they are not
     # relevant since they don't show up in any other build so it's better to suppress them until we
@@ -226,16 +235,19 @@ function(enable_static_checkers target)
     # on real builds.
     foreach(_flag -Wno-missing-braces -Wno-unneeded-internal-declaration)
       list(FIND CUDF_CXX_FLAGS "${_flag}" _flag_index)
+
       if(_flag_index EQUAL -1)
         list(APPEND CUDF_CXX_FLAGS ${_flag})
       endif()
     endforeach()
+
     set(CUDF_CXX_FLAGS
         "${CUDF_CXX_FLAGS}"
         PARENT_SCOPE
     )
     set_target_properties(${target} PROPERTIES CXX_INCLUDE_WHAT_YOU_USE "${IWYU_EXE}")
   endif()
+
   foreach(file IN LISTS _LINT_SKIPPED_FILES)
     set_source_files_properties(${file} PROPERTIES SKIP_LINTING ON)
   endforeach()
@@ -279,32 +291,45 @@ create_logger_macros(CUDF "cudf::default_logger()" include/cudf)
 
 # find jitify
 include(cmake/thirdparty/get_jitify.cmake)
+
 # find NVTX
 include(cmake/thirdparty/get_nvtx.cmake)
+
 # find nvCOMP
 include(cmake/thirdparty/get_nvcomp.cmake)
+
 # find CCCL before rmm so that we get cudf's patched version of CCCL
 include(cmake/thirdparty/get_cccl.cmake)
+
 # find rmm
 include(cmake/thirdparty/get_rmm.cmake)
+
 # find flatbuffers
 include(cmake/thirdparty/get_flatbuffers.cmake)
+
 # find dlpack
 include(cmake/thirdparty/get_dlpack.cmake)
+
 # find cuCollections, should come after including CCCL
 include(cmake/thirdparty/get_cucollections.cmake)
+
 # find or install GoogleTest
 if(CUDF_BUILD_TESTUTIL)
   include(cmake/thirdparty/get_gtest.cmake)
 endif()
+
 # preprocess jitify-able kernels
 include(cmake/Modules/JitifyPreprocessKernels.cmake)
+
 # find KvikIO
 include(cmake/thirdparty/get_kvikio.cmake)
+
 # find nanoarrow
 include(cmake/thirdparty/get_nanoarrow.cmake)
+
 # find thread_pool
 include(cmake/thirdparty/get_thread_pool.cmake)
+
 # find zstd
 include(cmake/thirdparty/get_zstd.cmake)
 
@@ -327,7 +352,6 @@ endif()
 
 # ##################################################################################################
 # * library targets -------------------------------------------------------------------------------
-
 add_library(
   cudf
   src/aggregation/aggregation.cpp
@@ -754,6 +778,7 @@ add_library(
   src/table/table.cpp
   src/table/table_device_view.cu
   src/table/table_view.cpp
+  src/text/deduplicate.cu
   src/text/detokenize.cu
   src/text/edit_distance.cu
   src/text/generate_ngrams.cu
@@ -857,16 +882,20 @@ set_target_properties(
 # flags if necessary.
 if(CUDF_CLANG_TIDY OR CUDF_IWYU)
   set(linters)
+
   if(CUDF_CLANG_TIDY)
     list(APPEND linters CLANG_TIDY)
   endif()
+
   if(CUDF_IWYU)
     list(APPEND linters IWYU)
   endif()
+
   enable_static_checkers(
     cudf SKIPPED_FILES src/io/comp/cpu_unbz2.cpp src/io/comp/brotli_dict.cpp ${linters}
   )
 endif()
+
 target_compile_options(
   cudf PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:${CUDF_CXX_FLAGS}>"
                "$<$<COMPILE_LANGUAGE:CUDA>:${CUDF_CUDA_FLAGS}>"
@@ -997,7 +1026,6 @@ add_library(cudf::cudf ALIAS cudf)
 
 # ##################################################################################################
 # * build cudftestutil ----------------------------------------------------------------------------
-
 if(CUDF_BUILD_TESTUTIL)
   add_library(
     cudftest_default_stream
@@ -1096,11 +1124,9 @@ if(CUDF_BUILD_TESTUTIL)
           tests/utilities/tdigest_utilities.cu
     DESTINATION src/cudftestutil/utilities
   )
-
 endif()
 
 # * build cudf_identify_stream_usage --------------------------------------------------------------
-
 if(CUDF_BUILD_STREAMS_TEST_UTIL)
   if(CUDA_STATIC_RUNTIME)
     message(
@@ -1117,6 +1143,7 @@ if(CUDF_BUILD_STREAMS_TEST_UTIL)
     add_library(
       ${_tgt} SHARED src/utilities/stacktrace.cpp tests/utilities/identify_stream_usage.cpp
     )
+
     if(CUDF_USE_PER_THREAD_DEFAULT_STREAM)
       target_compile_definitions(
         ${_tgt} PUBLIC CUDA_API_PER_THREAD_DEFAULT_STREAM CUDF_USE_PER_THREAD_DEFAULT_STREAM
@@ -1135,9 +1162,11 @@ if(CUDF_BUILD_STREAMS_TEST_UTIL)
     )
     target_include_directories(${_tgt} PRIVATE "$<BUILD_INTERFACE:${CUDF_SOURCE_DIR}/include>")
     target_link_libraries(${_tgt} PUBLIC CUDA::cudart rmm::rmm)
+
     if(CUDF_BUILD_STACKTRACE_DEBUG)
       target_link_libraries(${_tgt} PRIVATE cudf_backtrace)
     endif()
+
     rapids_cuda_set_runtime(${_tgt} USE_STATIC ${CUDA_STATIC_RUNTIME})
     add_library(cudf::${_tgt} ALIAS ${_tgt})
 
@@ -1149,7 +1178,6 @@ endif()
 
 # ##################################################################################################
 # * add tests -------------------------------------------------------------------------------------
-
 if(CUDF_BUILD_TESTS)
   # include CTest module -- automatically calls enable_testing()
   include(CTest)
@@ -1167,7 +1195,6 @@ endif()
 
 # ##################################################################################################
 # * add benchmarks --------------------------------------------------------------------------------
-
 if(CUDF_BUILD_BENCHMARKS)
   # Find or install GoogleBench
   include(${rapids-cmake-dir}/cpm/gbench.cmake)
@@ -1198,6 +1225,7 @@ install(FILES ${CUDF_BINARY_DIR}/include/cudf/version_config.hpp
 )
 
 set(_components_export_string)
+
 if(TARGET cudftestutil)
   install(
     TARGETS cudftest_default_stream cudftestutil cudftestutil_impl

cpp/include/doxygen_groups.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021-2024, NVIDIA CORPORATION.
+ * Copyright (c) 2021-2025, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -175,6 +175,7 @@
  *   @defgroup nvtext_replace Replacing
  *   @defgroup nvtext_minhash MinHashing
  *   @defgroup nvtext_jaccard Jaccard Index
+ *   @defgroup nvtext_dedup Deduplication
  * @}
  * @defgroup utility_apis Utilities
  * @{

cpp/include/nvtext/deduplicate.hpp

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cudf/column/column.hpp>
+#include <cudf/strings/strings_column_view.hpp>
+#include <cudf/utilities/export.hpp>
+#include <cudf/utilities/memory_resource.hpp>
+
+#include <rmm/cuda_stream_view.hpp>
+#include <rmm/device_uvector.hpp>
+
+//! NVText APIs
+namespace CUDF_EXPORT nvtext {
+/**
+ * @addtogroup nvtext_dedup
+ * @{
+ * @file
+ */
+
+/**
+ * @brief Returns duplicate strings found in the given input
+ *
+ * The internal implementation creates a suffix array of the input which
+ * requires ~10x the input size for temporary memory.
+ *
+ * The output includes any strings of at least `min_width` bytes that
+ * appear more than once in the entire input.
+ *
+ * @throw std::invalid_argument If `min_width` <= 8
+ * @throw std::invalid_argument If `min_width` is greater than the input chars size
+ * @throw std::invalid_argument If the `input` chars size is greater than 2GB
+ *
+ * @param input Strings column to identify duplicates
+ * @param min_width Minimum number of bytes that must match to identify a duplicate
+ * @param stream CUDA stream used for device memory operations and kernel launches
+ * @param mr Device memory resource used to allocate the returned column's device memory
+ * @return New strings column with updated strings
+ */
+std::unique_ptr<cudf::column> substring_duplicates(
+  cudf::strings_column_view const& input,
+  cudf::size_type min_width,
+  rmm::cuda_stream_view stream      = cudf::get_default_stream(),
+  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
+
+/**
+ * @brief Builds a suffix array for the input strings column
+ *
+ * The internal implementation creates a suffix array of the input which
+ * requires ~4x the input size for temporary memory. The output is an additional
+ * 4x of the input size.
+ *
+ * @throw std::invalid_argument If `min_width` is greater than the input chars size
+ * @throw std::invalid_argument If the `input` chars size is greater than 2GB
+ *
+ * @param input Strings column to build suffix array for
+ * @param min_width Minimum number of bytes that must match to identify a duplicate
+ * @param stream CUDA stream used for device memory operations and kernel launches
+ * @param mr Device memory resource used to allocate the returned column's device memory
+ * @return Sorted suffix array and corresponding sizes
+ */
+std::unique_ptr<rmm::device_uvector<cudf::size_type>> build_suffix_array(
+  cudf::strings_column_view const& input,
+  cudf::size_type min_width,
+  rmm::cuda_stream_view stream      = cudf::get_default_stream(),
+  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
+
+/** @} */  // end of group
+}  // namespace CUDF_EXPORT nvtext