mhp::halo tests working on device memory and on ishmem backend (#724)

Author: lslusarczyk

CMakeLists.txt

@@ -198,6 +198,7 @@ function(add_mhp_ctest_impl)
   endif()
 
   if(AMC_DRLOGS)
+    set(AMC_TEST_NAME "${AMC_TEST_NAME}-log")
     set(drlogs_param --log --logprefix=${AMC_TEST_NAME})
   endif()
 
@@ -219,7 +220,7 @@ function(add_mhp_ctest_impl)
       ${MPIEXEC_PREFLAGS} ${extra_mpiflags} ${wrapper_script} ./${AMC_NAME}
       ${drlogs_param} ${sycl_param} ${AMC_TARGS} COMMAND_EXPAND_LISTS)
 
-  if(NOT AMC_GDB)
+  if(NOT AMC_GDB AND NOT AMC_DRLOGS)
     set_property(TEST ${AMC_TEST_NAME} PROPERTY LABELS TESTLABEL MHP)
   endif()
   add_dependencies(mhp-all-tests ${AMC_NAME})
@@ -228,6 +229,8 @@ endfunction()
 function(add_mhp_ctest)
   add_mhp_ctest_impl(${ARGN})
   add_mhp_ctest_impl(${ARGN} GDB)
+  add_mhp_ctest_impl(${ARGN} GDB DRLOGS)
+  add_mhp_ctest_impl(${ARGN} DRLOGS)
 endfunction()
 
 if(ENABLE_ISHMEM)

include/dr/mhp/halo.hpp

@@ -64,15 +64,18 @@ template <typename Group> class halo_impl {
 
   /// Begin a halo exchange
   void exchange_begin() {
-    drlog.debug("Halo exchange begin\n");
+    DRLOG("Halo exchange receiving");
     receive(halo_groups_);
+    DRLOG("Halo exchange sending");
     send(owned_groups_);
+    DRLOG("Halo exchange begin finished");
   }
 
   /// Complete a halo exchange
   void exchange_finalize() {
+    DRLOG("Halo exchange finalize started");
     reduce_finalize();
-    drlog.debug("Halo exchange finalize\n");
+    DRLOG("Halo exchange finalize finished");
   }
 
   void exchange() {
@@ -92,7 +95,7 @@ template <typename Group> class halo_impl {
       int completed;
       MPI_Waitany(rng::size(requests_), requests_.data(), &completed,
                   MPI_STATUS_IGNORE);
-      drlog.debug("Completed: {}\n", completed);
+      DRLOG("reduce_finalize(op) waitany completed: {}", completed);
       auto &g = *map_[completed];
       if (g.receive && g.buffered) {
         g.unpack(op);
@@ -106,7 +109,7 @@ template <typename Group> class halo_impl {
       int completed;
       MPI_Waitany(rng::size(requests_), requests_.data(), &completed,
                   MPI_STATUS_IGNORE);
-      drlog.debug("Completed: {}\n", completed);
+      DRLOG("reduce_finalize() waitany completed: {}", completed);
       auto &g = *map_[completed];
       if (g.receive && g.buffered) {
         g.unpack();
@@ -146,7 +149,7 @@ template <typename Group> class halo_impl {
     for (auto &g : sends) {
       g.pack();
       g.receive = false;
-      drlog.debug("Sending: {}\n", g.request_index);
+      DRLOG("sending: {}", g.request_index);
       comm_.isend(g.data_pointer(), g.data_size(), g.rank(), g.tag(),
                   &requests_[g.request_index]);
     }
@@ -155,7 +158,7 @@ template <typename Group> class halo_impl {
   void receive(std::vector<Group> &receives) {
     for (auto &g : receives) {
       g.receive = true;
-      drlog.debug("Receiving: {}\n", g.request_index);
+      DRLOG("receiving: {}", g.request_index);
       comm_.irecv(g.data_pointer(), g.data_size(), g.rank(), g.tag(),
                   &requests_[g.request_index]);
     }

test/gtest/mhp/CMakeLists.txt

@@ -56,8 +56,7 @@ add_executable(
 
 add_executable(mhp-quick-test
   mhp-tests.cpp
-  ../common/exclusive_scan.cpp
-  ../common/inclusive_scan.cpp
+  halo.cpp
   )
 # cmake-format: on
 
@@ -101,6 +100,10 @@ if(ENABLE_SYCL)
 
   add_mhp_ctest(NAME mhp-quick-test NPROC 1 SYCL)
   add_mhp_ctest(NAME mhp-quick-test NPROC 2 SYCL)
+  add_mhp_ctest(
+    NAME mhp-quick-test NPROC 1 OFFLOAD SYCL TARGS --device-memory)
+  add_mhp_ctest(
+    NAME mhp-quick-test NPROC 2 OFFLOAD SYCL TARGS --device-memory)
 
   add_mhp_ctest(
     NAME mhp-tests NPROC 2 OFFLOAD SYCL TARGS --device-memory

test/gtest/mhp/halo.cpp

@@ -6,17 +6,17 @@
 
 template <typename T> class Halo : public testing::Test {};
 
-// segfault with ISHMEM
-TYPED_TEST_SUITE(Halo, AllTypesWithoutIshmem);
+TYPED_TEST_SUITE(Halo, AllTypes);
 
 template <typename DV>
 void local_is_accessible_in_halo_region(const int halo_prev,
                                         const int halo_next) {
-  if (options.count("device-memory")) {
-    return;
-  }
+
   DV dv(6, dr::mhp::distribution().halo(halo_prev, halo_next));
+  DRLOG("local_is_accessible_in_halo_region TEST START, prev:{}, next:{}",
+        halo_prev, halo_next);
   iota(dv, 0);
+  DRLOG("exchange start");
   dv.halo().exchange();
 
   // arrays below is function depending on size of communicator-1
@@ -58,16 +58,24 @@ void local_is_accessible_in_halo_region(const int halo_prev,
   auto first_legal_idx = std::max(0, first_local_index___[c] - halo_prev);
   auto first_illegal_idx = std::min(6, first_nonlocal_index[c] + halo_next);
 
-  dr::drlog.debug(
-      "checking access to idx between first legal {} and first illegal {}\n",
-      first_legal_idx, first_illegal_idx);
+  DRLOG("checking access to idx between first legal {} and first illegal {}, "
+        "c:{}",
+        first_legal_idx, first_illegal_idx, c);
 
   for (int idx = first_legal_idx; idx < first_illegal_idx; ++idx) {
-    dr::drlog.debug("checking idx:{}\n", idx);
-    EXPECT_TRUE((dv.begin() + idx).local() != nullptr);
-    EXPECT_EQ(*(dv.begin() + idx).local(), idx);
+    typename DV::value_type *local_ptr = (dv.begin() + idx).local();
+    EXPECT_TRUE(local_ptr != nullptr);
+    typename DV::value_type value_on_host;
+
+    if (dr::mhp::use_sycl())
+      dr::mhp::__detail::sycl_copy(local_ptr, &value_on_host);
+    else
+      value_on_host = *local_ptr;
+
+    DRLOG("checking idx:{}", idx);
+    EXPECT_EQ(value_on_host, idx);
   }
-  dr::drlog.debug("checks ok\n");
+  DRLOG("checks ok");
 
   // although assertions indeed happen, but they are not caught by EXPECT_DEATH
   //  if (first_illegal_idx < 6) {

test/gtest/mhp/xhp-tests.hpp

@@ -39,8 +39,8 @@ inline void fence_on(auto &&obj) { obj.fence(); }
 // minimal testing for quick builds
 #ifdef DRISHMEM
 using AllTypes =
-    ::testing::Types<dr::mhp::distributed_vector<int>,
-                     dr::mhp::distributed_vector<int, dr::mhp::IshmemBackend>>;
+    ::testing::Types<dr::mhp::distributed_vector<int, dr::mhp::IshmemBackend>,
+                     dr::mhp::distributed_vector<int>>;
 using IshmemTypes =
     ::testing::Types<dr::mhp::distributed_vector<int, dr::mhp::IshmemBackend>>;
 #else