Detect unexpected directories in the cache (#1481)

Scan cache folder for the unexpected directories as their presence
means that the cache was changed outside of Data SDK, or the wrong
cache was used in a first place.

Relates-To: OLPEDGE-2857

Signed-off-by: Andrey Kashcheev <ext-andrey.kashcheev@here.com>

Author: andrey-kashcheev

olp-cpp-sdk-core/include/olp/core/utils/Dir.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2023 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,9 @@ class CORE_API Dir {
   /// An alias for the filter function.
   using FilterFunction = std::function<bool(const std::string&)>;
 
+  /// An alias for the path callback function.
+  using PathCallback = std::function<void(const std::string&)>;
+
   /**
    * @brief Checks whether a directory exists.
    *
@@ -143,6 +146,15 @@ class CORE_API Dir {
    * path.
    */
   static bool IsReadOnly(const std::string& path);
+
+  /**
+   * @brief Iterates through all directories in the provided path and calls the
+   * provided callback function for each directory.
+   *
+   * @param path The path of the root directory.
+   * @param path_fn The callback function.
+   */
+  static void ForEachDirectory(const std::string& path, PathCallback path_fn);
 };
 
 }  // namespace utils

olp-cpp-sdk-core/src/cache/DiskCache.cpp

@@ -204,6 +204,23 @@ OpenResult DiskCache::Open(const std::string& data_path,
     }
   }
 
+  // Check cache path for unexpected directories
+  const std::vector<std::string> expected_dirs = {disk_cache_path_ +
+                                                  kLevelDbLostFolder};
+  bool unexpected_dirs = false;
+  utils::Dir::ForEachDirectory(disk_cache_path_, [&](const std::string& dir) {
+    if (std::find(expected_dirs.begin(), expected_dirs.end(), dir) ==
+        expected_dirs.end()) {
+      OLP_SDK_LOG_WARNING_F(
+          kLogTag, "Open: unexpected directory found, path='%s'", dir.c_str());
+      unexpected_dirs = true;
+    }
+  });
+
+  if (unexpected_dirs) {
+    return OpenResult::Fail;
+  }
+
   enforce_immediate_flush_ = settings.enforce_immediate_flush;
 
   max_size_ = settings.max_disk_storage;

olp-cpp-sdk-core/src/utils/Dir.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2023 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -530,5 +530,79 @@ bool Dir::IsReadOnly(const std::string& path) {
 #endif
 }
 
+void Dir::ForEachDirectory(const std::string& path, PathCallback path_fn) {
+#if defined(_WIN32) && !defined(__MINGW32__)
+
+  WIN32_FIND_DATAA find_data;
+  std::string current_path = path + "\\*.*";
+
+  HANDLE handle = FindFirstFileA(current_path.c_str(), &find_data);
+
+  if (handle != INVALID_HANDLE_VALUE) {
+    do {
+      if (strcmp(find_data.cFileName, ".") == 0 ||
+          strcmp(find_data.cFileName, "..") == 0) {
+        continue;
+      }
+
+      if (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+        current_path = path + "\\" + find_data.cFileName;
+        path_fn(current_path);
+        ForEachDirectory(current_path, path_fn);
+      }
+    } while (FindNextFileA(handle, &find_data) != 0);
+
+    FindClose(handle);
+  }
+
+#else
+
+  std::queue<std::string> recursive_queue;
+  recursive_queue.push(path);
+
+  auto iterate_directory = [&](const std::string& path) {
+    auto* dir = ::opendir(path.c_str());
+    if (dir == nullptr) {
+      return;
+    }
+
+    struct ::dirent* entry = nullptr;
+    while ((entry = ::readdir(dir)) != nullptr) {
+      const char* entry_name = entry->d_name;
+
+      if (strcmp(entry_name, ".") == 0 || strcmp(entry_name, "..") == 0) {
+        continue;
+      }
+
+      std::string full_path = path + "/" + entry_name;
+
+#ifdef __APPLE__
+      struct ::stat path_stat;
+      if (::lstat(full_path.c_str(), &path_stat) == 0) {
+#else
+      struct ::stat64 path_stat;
+      if (::lstat64(full_path.c_str(), &path_stat) == 0) {
+#endif
+        if (S_ISDIR(path_stat.st_mode)) {
+          path_fn(full_path);
+          recursive_queue.push(std::move(full_path));
+        }
+      } else if (errno != ENOENT) {
+        // Ignore ENOENT errors as its a common case, e.g. cache compaction.
+        break;
+      }
+    }
+
+    ::closedir(dir);
+  };
+
+  while (!recursive_queue.empty()) {
+    iterate_directory(recursive_queue.front());
+    recursive_queue.pop();
+  }
+
+#endif
+}
+
 }  // namespace utils
 }  // namespace olp

olp-cpp-sdk-core/tests/cache/DefaultCacheTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2021 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 #include <fstream>
 #include <string>
 #include <thread>
+#include <tuple>
 
 #include <gtest/gtest.h>
 
@@ -684,6 +685,54 @@ TEST(DefaultCacheTest, OpenTypeCache) {
     ASSERT_FALSE(cache.Contains(key1));
     ASSERT_FALSE(cache.Contains(key2));
   }
+
+  {
+    SCOPED_TRACE("Additional directories");
+
+    constexpr auto kExpectedDirResult =
+        olp::cache::DefaultCache::StorageOpenResult::Success;
+    constexpr auto kUnexpectedDirResult =
+        olp::cache::DefaultCache::StorageOpenResult::OpenDiskPathFailure;
+
+    auto scenarios = {
+        std::make_tuple("/lost", kExpectedDirResult),
+        std::make_tuple("/lost/tmp", kUnexpectedDirResult),
+        std::make_tuple("/found", kUnexpectedDirResult),
+        std::make_tuple("/ARCHIVES", kUnexpectedDirResult),
+        std::make_tuple("/ARCHIVES/removed", kUnexpectedDirResult)};
+
+    for (auto& scenario : scenarios) {
+      {
+        SCOPED_TRACE("Mutable cache");
+
+        std::string dir_path = mutable_path + std::get<0>(scenario);
+        ASSERT_TRUE(olp::utils::Dir::Create(dir_path));
+
+        olp::cache::CacheSettings settings;
+        settings.disk_path_mutable = mutable_path;
+
+        olp::cache::DefaultCache cache(settings);
+        EXPECT_EQ(cache.Open(), std::get<1>(scenario));
+
+        olp::utils::Dir::Remove(dir_path);
+      }
+
+      {
+        SCOPED_TRACE("Protected cache");
+
+        std::string dir_path = protected_path + std::get<0>(scenario);
+        ASSERT_TRUE(olp::utils::Dir::Create(dir_path));
+
+        olp::cache::CacheSettings settings;
+        settings.disk_path_protected = protected_path;
+
+        olp::cache::DefaultCache cache(settings);
+        EXPECT_EQ(cache.Open(), std::get<1>(scenario));
+
+        olp::utils::Dir::Remove(dir_path);
+      }
+    }
+  }
 }
 
 struct TestParameters {