diff --git a/app/main.cpp b/app/main.cpp index d239e25..c362a82 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -117,11 +117,12 @@ void Create(std::unordered_map args) { std::string L_small, R_small, R_stiched; bool save = false; bool leaveEmpty = false; + int distanceThreads = 1; // Default value - std::vector validArguments = {"-index-type", "-base-file", "-L", "-L-small", "-R", "-R-small", "-R-stiched", "-alpha", "-save", "-random-edges", "-connection-mode"}; + std::vector validArguments = {"-index-type", "-base-file", "-L", "-L-small", "-R", "-R-small", "-R-stiched", "-alpha", "-save", "-random-edges", "-connection-mode", "-distance-threads"}; for (auto arg : args) { if (std::find(validArguments.begin(), validArguments.end(), arg.first) == validArguments.end()) { - throw std::invalid_argument("Error: Invalid argument: " + arg.first + ". Valid arguments are: -index-type, -base-file, -L, -L-small, -R, -R-small, -R-stiched, -alpha, -save, -connection-mode"); + throw std::invalid_argument("Error: Invalid argument: " + arg.first + ". Valid arguments are: -index-type, -base-file, -L, -L-small, -R, -R-small, -R-stiched, -alpha, -save, -connection-mode, -distance-threads"); } } @@ -195,6 +196,10 @@ void Create(std::unordered_map args) { } } + if (args.find("-distance-threads") != args.end()) { + distanceThreads = std::stoi(args["-distance-threads"]); + } + if (indexType == "simple") { BaseVectors base_vectors = ReadVectorFile(baseFile); if (base_vectors.empty()) { @@ -203,7 +208,7 @@ void Create(std::unordered_map args) { } VamanaIndex> vamanaIndex = VamanaIndex>(); - vamanaIndex.createGraph(base_vectors, std::stof(alpha), std::stoi(L), std::stoi(R)); + vamanaIndex.createGraph(base_vectors, std::stof(alpha), std::stoi(L), std::stoi(R), distanceThreads, true); if (save) { if (!vamanaIndex.saveGraph(outputFile)) { @@ -223,7 +228,7 @@ void Create(std::unordered_map args) { if (indexType == "filtered") { FilteredVamanaIndex> index(filters); - index.createGraph(base_vectors, std::stoi(alpha), std::stoi(L), std::stoi(R), true, leaveEmpty); + index.createGraph(base_vectors, std::stoi(alpha), std::stoi(L), std::stoi(R), distanceThreads, true, leaveEmpty); if (save) { index.saveGraph(outputFile); @@ -231,7 +236,7 @@ void Create(std::unordered_map args) { } } else if (indexType == "stiched") { StichedVamanaIndex> index(filters); - index.createGraph(base_vectors, std::stof(alpha), std::stoi(L_small), std::stoi(R_small), std::stoi(R_stiched), true, leaveEmpty); + index.createGraph(base_vectors, std::stof(alpha), std::stoi(L_small), std::stoi(R_small), std::stoi(R_stiched), distanceThreads, true, leaveEmpty); if (save) { index.saveGraph(outputFile); diff --git a/include/FilteredVamanaIndex.h b/include/FilteredVamanaIndex.h index 3ba3e1c..c813f68 100644 --- a/include/FilteredVamanaIndex.h +++ b/include/FilteredVamanaIndex.h @@ -60,7 +60,7 @@ template class FilteredVamanaIndex : public VamanaIndex& P, const float& alpha, const unsigned int L, const unsigned int R, bool visualized = true, bool empty = true); + void createGraph(const std::vector& P, const float& alpha, const unsigned int L, const unsigned int R, unsigned int distance_threads = 1, bool visualized = true, bool empty = true); /** * @brief Load a graph from a file. Specifically this method is used to receive the contents of a Vamana Index Graph diff --git a/include/StichedVamanaIndex.h b/include/StichedVamanaIndex.h index f5935e9..343c0a7 100644 --- a/include/StichedVamanaIndex.h +++ b/include/StichedVamanaIndex.h @@ -31,7 +31,7 @@ template class StichedVamanaIndex : public FilteredVamanaInd * @param R An unsigned int parameter. */ void createGraph(const std::vector& P, const float& alpha, const unsigned int L_small, - const unsigned int R_small, const unsigned int R_stiched, bool visualized = true, bool empty = true); + const unsigned int R_small, const unsigned int R_stiched, unsigned int distance_threads, bool visualized = true, bool empty = true); }; diff --git a/include/VamanaIndex.h b/include/VamanaIndex.h index c533532..df3493f 100644 --- a/include/VamanaIndex.h +++ b/include/VamanaIndex.h @@ -49,8 +49,9 @@ template class VamanaIndex { * @brief Computes the distances between every node in the dataset and stores them in the distance matrix. * * @param visualize a boolean flag to visualize the progress of the computation + * @param numThreads the number of threads to use for computation */ - void computeDistances(const bool visualize = true); + void computeDistances(const bool visualize = true, const unsigned int numThreads = 1); public: @@ -102,7 +103,7 @@ template class VamanaIndex { * @param R the parameter R * */ - void createGraph(const std::vector& P, const float& alpha, const unsigned int L, const unsigned int& R, bool visualize = true, double** distanceMatrix = nullptr); + void createGraph(const std::vector& P, const float& alpha, const unsigned int L, const unsigned int& R, unsigned int distance_threads = 1, bool visualize = true, double** distanceMatrix = nullptr); /** * @brief Saves a specific graph into a file. Specifically this method is used to save the contents of a Vamana diff --git a/include/graphics.h b/include/graphics.h index 6466633..a149569 100644 --- a/include/graphics.h +++ b/include/graphics.h @@ -31,6 +31,7 @@ const std::string brightWhite = "\033[1;37m"; const std::string ProgressSymbol = "\u25AC"; const std::string RemainingSymbol = "\u25AC"; +const std::string tickSymbol = "\u2713"; /** diff --git a/src/Graphics/ProgressBar.cpp b/src/Graphics/ProgressBar.cpp index 1f95402..4645325 100644 --- a/src/Graphics/ProgressBar.cpp +++ b/src/Graphics/ProgressBar.cpp @@ -104,7 +104,7 @@ void displayProgressBar( } else if (current == total) { std::cout << " " << verticalLineSymbol << " " << yellow; - std::cout << brightGreen << "Done" << std::setw(14) << std::setfill(' ') << reset; + std::cout << brightGreen << "Done " << tickSymbol << std::setw(12) << std::setfill(' ') << reset; } // Display elapsed time @@ -137,8 +137,9 @@ void withProgress( auto startTime = std::chrono::steady_clock::now(); for (unsigned int i = start; i < end; i++) { - displayProgressBar(i - start + 1, total, message, startTime, barWidth); + displayProgressBar(i - start, total, message, startTime, barWidth); func(i); + displayProgressBar(i - start + 1, total, message, startTime, barWidth); } std::cout << std::endl; diff --git a/src/VIA/Algorithms/FilteredVamanaIndex.cpp b/src/VIA/Algorithms/FilteredVamanaIndex.cpp index c7c3322..1e1d405 100644 --- a/src/VIA/Algorithms/FilteredVamanaIndex.cpp +++ b/src/VIA/Algorithms/FilteredVamanaIndex.cpp @@ -79,7 +79,7 @@ FilteredVamanaIndex::getNodesWithCategoricalValueFilter(const Categori */ template void FilteredVamanaIndex::createGraph( - const std::vector& P, const float& alpha, const unsigned int L, const unsigned int R, bool visualized, bool empty) { + const std::vector& P, const float& alpha, const unsigned int L, const unsigned int R, unsigned int distance_threads, bool visualized, bool empty) { using Filter = CategoricalAttributeFilter; using GreedyResult = std::pair, std::set>; @@ -91,7 +91,7 @@ void FilteredVamanaIndex::createGraph( for (unsigned int i = 0; i < n; i++) { this->distanceMatrix[i] = new double[n]; } - this->computeDistances(); + this->computeDistances(true, distance_threads); this->G.setNodesCount(n); // Initialize G to an empty graph and get the medoid node diff --git a/src/VIA/Algorithms/StichedVamanaIndex.cpp b/src/VIA/Algorithms/StichedVamanaIndex.cpp index 1564b55..f4c9bf0 100644 --- a/src/VIA/Algorithms/StichedVamanaIndex.cpp +++ b/src/VIA/Algorithms/StichedVamanaIndex.cpp @@ -17,7 +17,7 @@ */ template void StichedVamanaIndex::createGraph(const std::vector& P, const float& alpha, const unsigned int L_small, - const unsigned int R_small, const unsigned int R_stiched, bool visualized, bool empty) { + const unsigned int R_small, const unsigned int R_stiched, unsigned int distance_threads, bool visualized, bool empty) { using Filter = CategoricalAttributeFilter; @@ -28,7 +28,7 @@ void StichedVamanaIndex::createGraph(const std::vector& P, c for (unsigned int i = 0; i < n; i++) { this->distanceMatrix[i] = new double[n]; } - this->computeDistances(); + this->computeDistances(true, distance_threads); // Initialize G = (V, E) to an empty graph this->G.setNodesCount(n); @@ -73,7 +73,7 @@ void StichedVamanaIndex::createGraph(const std::vector& P, c // Initialize the sub-index for the current filter and create its graph VamanaIndex subIndex; - subIndex.createGraph(Pf[filter], alpha, R_small, L_small, false, this->distanceMatrix); + subIndex.createGraph(Pf[filter], alpha, R_small, L_small, 1, false, this->distanceMatrix); for (unsigned int i = 0; i < subIndex.getGraph().getNodesCount(); i++) { diff --git a/src/VIA/Algorithms/VamanaIndex.cpp b/src/VIA/Algorithms/VamanaIndex.cpp index dc94a58..8bce7c8 100644 --- a/src/VIA/Algorithms/VamanaIndex.cpp +++ b/src/VIA/Algorithms/VamanaIndex.cpp @@ -4,12 +4,16 @@ #include #include +#include +#include #include #include #include #include #include +std::mutex distanceMutex; + /** * @brief Generates a random permutation of integers in a specified range. This function creates a vector * containing all integers from `start` to `end` and then shuffles them randomly to produce a random permutation. @@ -91,26 +95,53 @@ template void VamanaIndex::createRandomEdges(const /** * @brief Computes the distances between every node in the dataset and stores them in the distance matrix. */ -template void VamanaIndex::computeDistances(const bool visualize) { - - // Define a lambda function to compute the distances between two points - auto compute = [&](int i) { - for (unsigned int j = i; j < this->P.size(); j++) { - double dist = euclideanDistance(this->P.at(i), this->P.at(j)); - this->distanceMatrix[i][j] = dist; - this->distanceMatrix[j][i] = dist; +template +void VamanaIndex::computeDistances(const bool visualize, const unsigned int numThreads) { + + std::atomic progress(0); + auto startTime = std::chrono::steady_clock::now(); + + // Define a lambda function to compute the distances between points + auto compute = [&](int start, int end) { + for (int i = start; i < end; ++i) { + for (unsigned int j = i; j < this->P.size(); ++j) { + double dist = euclideanDistance(this->P.at(i), this->P.at(j)); + this->distanceMatrix[i][j] = dist; + this->distanceMatrix[j][i] = dist; + } + progress++; + if (visualize && progress % 100 == 0) { + std::lock_guard lock(distanceMutex); + displayProgressBar(progress, this->P.size(), "Computing Distances", startTime, 30); + } } }; - // Compute distances with or without visualization, depending on the visualize flag value - if (visualize) { - withProgress(0, this->P.size(), "Computing Distances", compute); - } else { - for (unsigned int i = 0; i < this->P.size(); i++) { - compute(i); + // Compute distances using multiple threads if numThreads > 1 or a single thread otherwise + if (numThreads > 1) { + std::vector threads; + int chunkSize = this->P.size() / numThreads; + for (unsigned int t = 0; t < numThreads; ++t) { + int start = t * chunkSize; + int end = (t == numThreads - 1) ? this->P.size() : start + chunkSize; + threads.emplace_back(compute, start, end); } - } + for (auto& thread : threads) { + thread.join(); + } + if (visualize) { + displayProgressBar(this->P.size(), this->P.size(), "Computing Distances", startTime, 30); + std::cout << std::endl; + } + } + else { + if (visualize) { + withProgress(0, this->P.size(), "Computing Distances", [&](int i) { compute(i, i + 1); }); + } else { + compute(0, this->P.size()); + } + } } /** @@ -131,7 +162,7 @@ template void VamanaIndex::computeDistances(const */ template void VamanaIndex::createGraph( -const std::vector& P, const float& alpha, const unsigned int L, const unsigned int& R, bool visualize, double** distanceMatrix) { + const std::vector& P, const float& alpha, const unsigned int L, const unsigned int& R, unsigned int distance_threads, bool visualize, double** distanceMatrix) { using GreedyResult = std::pair, std::set>; GreedyResult greedyResult; @@ -151,8 +182,11 @@ const std::vector& P, const float& alpha, const unsigned int L, const for (unsigned int i = 0; i < n; i++) { this->distanceMatrix[i] = new double[n]; } - this->computeDistances(visualize); + this->computeDistances(visualize, distance_threads); } + + // this->computeDistances(false); + this->G.setNodesCount(n); // Set the number of nodes in the graph, fill the nodes with the dataset points, and create random edges for the nodes this->G.setNodesCount(n); @@ -163,43 +197,38 @@ const std::vector& P, const float& alpha, const unsigned int L, const GraphNode s = findMedoid(this->G, visualize, 1000); std::vector sigma = generateRandomPermutation(0, n-1); - // Define a lambda function for the main loop process of the Vamana algorithm - auto loopProcess = [&](int i) { - + // Define a lambda function to process each node in the sigma permutation + auto processNode = [&](int i) { GraphNode* sigma_i_node = this->G.getNode(sigma.at(i)); vamana_t sigma_i = sigma_i_node->getData(); - // Run Greedy Search and Robust Prune for the current sigma[i] node and its neighbors greedyResult = GreedySearch(*this, s, this->P.at(sigma.at(i)), 1, L); RobustPrune(*this, *sigma_i_node, greedyResult.second, alpha, R); - // Get the neighbors of sigma[i] node and iterate over them to run Robust Prune for each one of them as well - for (auto j : *sigma_i_node->getNeighborsVector()) { + std::vector* sigma_i_neighbors = sigma_i_node->getNeighborsVector(); + for (auto j : *sigma_i_neighbors) { std::set outgoing; GraphNode* j_node = this->G.getNode(j.getIndex()); - // The outgoing set has to consist of the neighbors of j and the sigma[i] node itself for (auto neighbor : *j_node->getNeighborsVector()) { outgoing.insert(neighbor); } outgoing.insert(sigma_i); - // Check if the |N_out(j) union {sigma[i]}| > R and run Robust Prune accordingly - if (outgoing.size() > (unsigned int)R) { + if (outgoing.size() > (long unsigned int)R) { RobustPrune(*this, *j_node, outgoing, alpha, R); } else { j_node->addNeighbor(sigma_i); } } - }; - // Process each node with or without visualization + // Run the lambda process function if visualization is enabled, otherwise run it without progress visualization if (visualize) { - withProgress(0, n, "Creating Vamana", loopProcess); + withProgress(0, n, "Creating Vamana", processNode); } else { for (unsigned int i = 0; i < n; i++) { - loopProcess(i); + processNode(i); } }