remove hwloc, fix benchmark bugs

Author: scivision

CMakeLists.txt

@@ -36,19 +36,18 @@ find_package(MPI COMPONENTS C Fortran REQUIRED)
 
 include(cmake/gcd.cmake)  # math functions
 
-# --- external libraries:  HDF5, HWLOC
+# --- external libraries:  HDF5
 
 include(cmake/check_mpi.cmake)
 check_mpi_version()
 
-find_package(HDF5 1.10.2 COMPONENTS Fortran HL parallel REQUIRED)
+find_package(HDF5 COMPONENTS Fortran HL parallel REQUIRED)
+if(HDF5_VERSION VERSION_LESS 1.10.2)
+  message(WARNING "HDF5 >= 1.10.2 is needed for HDF5-MPI")
+endif()
 
 if(ENABLE_BENCHMARKS)
-  find_package(HWLOC 2.4 REQUIRED)
-  find_package(HWLOCfortran CONFIG REQUIRED)
-
   add_subdirectory(benchmark)
-  add_subdirectory(benchmark/test)
 endif()
 
 # --- HDF5-MPI object oriented API
@@ -62,7 +61,6 @@ endif()
 # additional Find*.cmake necessary
 install(FILES
 ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindHDF5.cmake
-${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/check_hdf5_mpi.f90
 DESTINATION cmake
 )
 

benchmark/CMakeLists.txt

@@ -1,9 +1,11 @@
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.20...3.24)
 
 project(h5mpi-benchmark
-LANGUAGES C Fortran
+LANGUAGES C CXX Fortran
 )
 
+set_directory_properties(PROPERTIES LABELS benchmark)
+
 cmake_path(SET CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake/Modules)
 
 find_package(MPI COMPONENTS Fortran REQUIRED)
@@ -14,8 +16,6 @@ if(NOT EXISTS ${PROJECT_BINARY_DIR}/.gitignore)
   file(WRITE ${PROJECT_BINARY_DIR}/.gitignore "*")
 endif()
 
-add_subdirectory(test)
-
 set(runner_os \"${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}\")
 configure_file(perf.in.f90 perf.f90 @ONLY)
 add_library(perf OBJECT
@@ -29,13 +29,21 @@ add_library(partition OBJECT partition.f90)
 
 add_library(cli OBJECT cli.f90)
 
-add_executable(runner frontend.f90)
-target_link_libraries(runner PRIVATE partition cli HWLOCfortran::hwloc_ifc)
-set_target_properties(runner PROPERTIES
-RUNTIME_OUTPUT_DIRECTORY $<TARGET_FILE_DIR:slab_mpi_write>
-)
+add_executable(runner frontend.f90 cpu.cpp)
+target_link_libraries(runner PRIVATE partition cli)
+# set_target_properties(runner PROPERTIES
+# RUNTIME_OUTPUT_DIRECTORY $<TARGET_FILE_DIR:slab_mpi_write>
+# )
+if(CMAKE_Fortran_COMPILER_ID MATCHES "^Intel")
+  set_target_properties(runner PROPERTIES LINKER_LANGUAGE Fortran)
+else()
+  set_target_properties(runner PROPERTIES LINKER_LANGUAGE CXX)
+endif()
 
 foreach(t cli kernel perf)
   set_target_properties(${t} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include)
   target_include_directories(${t} INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/include)
 endforeach()
+
+
+add_subdirectory(test)

benchmark/cpu.cpp

@@ -0,0 +1,176 @@
+// adapted from Kitware kwsys, with BSD 3-Clause license
+// https://gitlab.kitware.com/utils/kwsys/-/blob/master/SystemInformation.cxx
+
+// Tested with:
+// Windows (g++, clang++, icx, cl)
+// MacOS (g++, clang++, icpc)
+// Linux (g++, clang++, icpx)
+
+#include <vector>
+#include <cassert>
+#include <bitset>
+#include <limits>
+#include <string>
+#include <cstring>
+#include <set>
+#include <thread>
+
+#ifdef _WIN32
+#include <windows.h>
+#elif defined (__APPLE__)
+#include <sys/sysctl.h>
+#else
+#include <unistd.h>
+#endif
+
+extern "C" unsigned int cpu_count();
+
+std::string ExtractValueFromCpuInfoFile(std::string buffer, const char* word,
+  size_t& CurrentPositionInFile, size_t init = 0);
+
+
+unsigned int cpu_count(){
+
+  unsigned int NumberOfPhysicalCPU = 0;
+  unsigned int NumberOfLogicalCPU = 0;
+
+#ifdef _WIN32
+
+  typedef BOOL(WINAPI * GetLogicalProcessorInformationType)(
+    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
+  static GetLogicalProcessorInformationType pGetLogicalProcessorInformation =
+    (GetLogicalProcessorInformationType)GetProcAddress(
+      GetModuleHandleW(L"kernel32"), "GetLogicalProcessorInformation");
+
+  if (!pGetLogicalProcessorInformation) {
+    return 0;
+  }
+
+  std::vector<SYSTEM_LOGICAL_PROCESSOR_INFORMATION> ProcInfo;
+  {
+    DWORD Length = 0;
+    DWORD rc = pGetLogicalProcessorInformation(nullptr, &Length);
+    assert(rc == 0);
+    assert(GetLastError() == ERROR_INSUFFICIENT_BUFFER);
+    ProcInfo.resize(Length / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION));
+    rc = pGetLogicalProcessorInformation(&ProcInfo[0], &Length);
+    assert(rc != 0);
+  }
+
+  typedef std::vector<SYSTEM_LOGICAL_PROCESSOR_INFORMATION>::iterator
+    pinfoIt_t;
+  for (pinfoIt_t it = ProcInfo.begin(); it != ProcInfo.end(); ++it) {
+    SYSTEM_LOGICAL_PROCESSOR_INFORMATION PInfo = *it;
+    if (PInfo.Relationship != RelationProcessorCore) {
+      continue;
+    }
+
+    std::bitset<std::numeric_limits<ULONG_PTR>::digits> ProcMask(
+      (unsigned long long)PInfo.ProcessorMask);
+    unsigned int count = (unsigned int)ProcMask.count();
+    if (count == 0) { // I think this should never happen, but just to be safe.
+      continue;
+    }
+    NumberOfPhysicalCPU++;
+    NumberOfLogicalCPU += (unsigned int)count;
+  }
+
+#elif defined(__linux)
+
+  std::string buffer;
+
+  FILE* fd = fopen("/proc/cpuinfo", "r");
+  if (!fd) return 0;
+
+  size_t fileSize = 0;
+  while (!feof(fd)) {
+    buffer += static_cast<char>(fgetc(fd));
+    fileSize++;
+  }
+  fclose(fd);
+  buffer.resize(fileSize - 2);
+  // Number of logical CPUs (combination of multiple processors, multi-core
+  // and SMT)
+  size_t pos = buffer.find("processor\t");
+  while (pos != std::string::npos) {
+    NumberOfLogicalCPU++;
+    pos = buffer.find("processor\t", pos + 1);
+  }
+
+  // Count sockets.
+  size_t CurrentPositionInFile;
+  std::set<int> PhysicalIDs;
+  std::string idc = ExtractValueFromCpuInfoFile(buffer, "physical id", CurrentPositionInFile);
+  while (CurrentPositionInFile != std::string::npos) {
+    int id = atoi(idc.c_str());
+    PhysicalIDs.insert(id);
+    idc = ExtractValueFromCpuInfoFile(buffer, "physical id",
+      CurrentPositionInFile, CurrentPositionInFile + 1);
+  }
+
+  uint64_t NumberOfSockets = PhysicalIDs.size();
+  NumberOfSockets = std::max(NumberOfSockets, (uint64_t)1);
+  // Physical ids returned by Linux don't distinguish cores.
+  // We want to record the total number of cores in this->NumberOfPhysicalCPU
+  // (checking only the first proc)
+  std::string Cores = ExtractValueFromCpuInfoFile(buffer, "cpu cores", CurrentPositionInFile);
+  if (Cores.empty()) {
+    // Linux Sparc is different
+    Cores = ExtractValueFromCpuInfoFile(buffer, "ncpus probed", CurrentPositionInFile);
+  }
+  auto NumberOfCoresPerSocket = (unsigned int)atoi(Cores.c_str());
+  NumberOfCoresPerSocket = std::max(NumberOfCoresPerSocket, 1u);
+  NumberOfPhysicalCPU = NumberOfCoresPerSocket * (unsigned int)NumberOfSockets;
+
+#elif defined(__APPLE__)
+
+  int N;
+  size_t size = sizeof(N);
+
+  if (sysctlbyname("hw.physicalcpu", &N, &size, nullptr, 0) == 0)
+    NumberOfPhysicalCPU = N;
+
+#elif defined(_SC_NPROCESSORS_ONLN)
+
+ long N = sysconf(_SC_NPROCESSORS_ONLN);
+ if (N > 0)
+    NumberOfPhysicalCPU = static_cast<unsigned int>(N);
+
+#else
+  NumberOfPhysicalCPU = std::thread::hardware_concurrency();
+#endif
+
+   return NumberOfPhysicalCPU;
+
+}
+
+/** Extract a value from the CPUInfo file */
+std::string ExtractValueFromCpuInfoFile(std::string buffer, const char* word,
+  size_t & CurrentPositionInFile, size_t init)
+{
+
+  size_t pos = buffer.find(word, init);
+  if (pos != std::string::npos) {
+    CurrentPositionInFile = pos;
+    pos = buffer.find(':', pos);
+    size_t pos2 = buffer.find('\n', pos);
+    if (pos != std::string::npos && pos2 != std::string::npos) {
+      // It may happen that the beginning matches, but this is still not the
+      // requested key.
+      // An example is looking for "cpu" when "cpu family" comes first. So we
+      // check that
+      // we have only spaces from here to pos, otherwise we search again.
+      for (size_t i = CurrentPositionInFile + strlen(word); i < pos;
+           ++i) {
+        if (buffer[i] != ' ' && buffer[i] != '\t') {
+          return ExtractValueFromCpuInfoFile(buffer, word, CurrentPositionInFile, pos2);
+        }
+      }
+      buffer.erase(0, pos + 2);
+      buffer.resize(pos2 - pos - 2);
+      return buffer;
+    }
+  }
+  CurrentPositionInFile = std::string::npos;
+  return "";
+}

benchmark/frontend.f90

@@ -1,12 +1,19 @@
 program frontend
 
 use, intrinsic :: iso_fortran_env, only: compiler_version
-use hwloc_ifc, only : get_cpu_count
+use, intrinsic :: iso_c_binding, only : C_INT
 use partition, only : max_gcd
 use cli, only : get_cli, get_simsize
 
 implicit none (type, external)
 
+interface
+integer(c_int) function cpu_count() bind(c, name="cpu_count")
+import c_int
+end function cpu_count
+end interface
+
+
 integer :: lid, lx1, lx2, lx3, Ncpu, ierr, Nrun, i, comp_lvl
 character(2000) :: buf, exe, mpiexec, outfn, refh5fn
 character(:), allocatable :: cmd, args
@@ -68,7 +75,7 @@ program frontend
 inquire(file=exe, exist=exists)
 if(.not. exists) error stop trim(exe) // " is not a file."
 
-if(Ncpu < 1) Ncpu = get_cpu_count()
+if(Ncpu < 1) Ncpu = cpu_count()
 lid = max_gcd(lx2, Ncpu)
 
 print '(A,I0)', 'MPI images: ', lid

benchmark/test/CMakeLists.txt

@@ -177,16 +177,16 @@ set(pyargs ${PROJECT_SOURCE_DIR}/scripts/bench_slab.py --datadir ${CMAKE_CURRENT
 add_test(NAME PythonRunner COMMAND Python::Interpreter ${pyargs})
 set_tests_properties(PythonRunner PROPERTIES
 FIXTURES_SETUP PyRun_fxt
-LABELS python
-TIMEOUT 60
-DISABLED ${PySkip}
 RUN_SERIAL true
 )
 
 add_test(NAME Plotter COMMAND Python::Interpreter ${PROJECT_SOURCE_DIR}/scripts/bench_plot.py --datadir ${CMAKE_CURRENT_BINARY_DIR} -lx ${lx3})
 set_tests_properties(Plotter PROPERTIES
 FIXTURES_REQUIRED PyRun_fxt
+)
+
+set_tests_properties(PythonRunner Plotter PROPERTIES
 LABELS python
-TIMEOUT 30
+TIMEOUT 60
 DISABLED ${PySkip}
 )

benchmark/test/slab_mpi_read.f90

@@ -148,8 +148,10 @@ program read_slab_mpi
 !> RESULTS
 
 if(mpi_id == mpi_root_id) then
+  call h5%open(trim(refh5fn), action="r", mpi=.false.)
   call print_timing(Nmpi, h5%comp_lvl, storage_size(A3), int(dims_full), t_elapsed, h5%filesize(), debug, &
     trim(h5fn) // ".read_stat.h5")
+  call h5%close()
 endif
 
 if (debug) print '(a,i0)', "mpi finalize: worker: ", mpi_id

benchmark/test/slab_mpi_serial_read.f90

@@ -147,17 +147,12 @@ program read_slab_mpi_root
   allocate(t3(lx1, lx2, lx3))
   call h5%open(trim(refh5fn), action="r", mpi=.false.)
   call h5%read("/A3", t3)
-  call h5%close()
 
   if (any(abs(t3 - A3) > 0.01)) error stop "3D ref mismatch " // trim(refh5fn) // " /= " // trim(h5fn)
-endif
-
 
-!> RESULTS
-
-if(mpi_id == mpi_root_id) then
   call print_timing(Nmpi, h5%comp_lvl, storage_size(A3), int(dims_full), t_elapsed, h5%filesize(), debug, &
     trim(h5fn) // ".read_stat.h5")
+  call h5%close()
 endif
 
 if (debug) print '(a,i0)', "mpi finalize: worker: ", mpi_id

benchmark/test/slab_mpi_serial_write.f90

@@ -177,21 +177,17 @@ program write_slab_mpi_root
 
   call h5%open(trim(h5fn), action="r", mpi=.false.)
   call h5%read("/A3", ts3)
-  call h5%close()
 
   if (any(abs(ts3 - S3) > 0.01)) then
     write(stderr,'(a,i0,1x,i0)') "ERROR: 3D disk vs. memory mismatch."
     write(stderr,'(a,100f5.1)') "disk: ", ts3
     write(stderr,'(a,100f5.1)') "memory: ", S3
     error stop trim(h5fn)
   endif
-endif
-
-!> RESULTS
 
-if(mpi_id == mpi_root_id) then
   call print_timing(Nmpi, h5%comp_lvl, storage_size(S3), int([lx1, lx2, lx3]), t_elapsed, h5%filesize(), debug, &
     trim(h5fn) // ".write_stat.h5")
+  call h5%close()
 endif
 
 if (debug) print '(a,i0)', "mpi finalize: worker: ", mpi_id

benchmark/test/slab_mpi_write.f90

@@ -185,8 +185,10 @@ program write_slab_mpi
 !> RESULTS
 
 if(mpi_id == mpi_root_id) then
+  call h5%open(trim(h5fn), action="r", mpi=.false.)
   call print_timing(Nmpi, h5%comp_lvl, storage_size(S3), [lx1, lx2, lx3], t_elapsed, h5%filesize(), debug, &
   trim(h5fn) // ".write_stat.h5")
+  call h5%close()
 endif
 
 if (debug) print '(a,i0)', "mpi finalize: worker: ", mpi_id

benchmark/test/slab_serial_read.f90

@@ -93,7 +93,9 @@ program read_slab_serial
 
 !> RESULTS
 
+call h5%open(trim(h5fn), action="r", mpi=.false.)
 call print_timing(1, h5%comp_lvl, real_bits, int(dims_full), t_elapsed, h5%filesize(), debug, &
 trim(h5fn) // ".read_stat.h5")
+call h5%close()
 
 end program