WIP: YAC-coupling integration in Elmer (colouring and mpi_handshake)

CMakeLists.txt

@@ -510,6 +510,15 @@ IF (WITH_XIOS)
     INCLUDE_DIRECTORIES(${XIOS_INCLUDE_DIR})
     LINK_DIRECTORIES(${XIOS_LIBRARIES})
     ADD_DEFINITIONS(-DHAVE_XIOS)
+
+    INCLUDE(testMPIHandshake)
+
+    MESSAGE(STATUS "  XIOS_HAS_MPI_HANDSHAKE: " "${XIOS_HAS_MPI_HANDSHAKE}")
+
+    IF(XIOS_HAS_MPI_HANDSHAKE)
+      ADD_DEFINITIONS(-DXIOS_HAS_MPI_HANDSHAKE)
+    ENDIF(XIOS_HAS_MPI_HANDSHAKE)
+
   ELSE(XIOS_FOUND)
     MESSAGE(STATUS "  ${XIOS_FAILMSG}")
   ENDIF(XIOS_FOUND)
@@ -536,6 +545,16 @@ IF (WITH_YAC)
                       "with -DWITH_NETCDF=ON and make sure NetCDF is found.")
   ENDIF()
 
+  MESSAGE(STATUS "  XIOS_HAS_MPI_HANDSHAKE: " "${XIOS_HAS_MPI_HANDSHAKE}")
+
+  IF(WITH_XIOS)
+    IF(NOT XIOS_HAS_MPI_HANDSHAKE)
+      MESSAGE(FATAL_ERROR "Building WITH_YAC and WITH_XIOS requires "
+                          "XIOS_HAS_MPI_HANDSHAKE. Please rebuild XIOS with "
+                          "MPI handshake support.")
+    ENDIF()
+  ENDIF()
+
   FIND_PACKAGE(YAXT)
 
   MESSAGE(STATUS "  YAXT_FOUND:     " "${YAXT_FOUND}")

cmake/Modules/FindYAC.cmake

@@ -51,9 +51,7 @@ IF (YAC_INCLUDE_DIR AND YAC_LIBRARY)
   # List of YAC static libraries
   SET(YACLIB_NAMES
     libyac.a
-    libyac_utils.a
     libyac_core.a
-    libyac_clapack.a
     libyac_mci.a
     libyac_mtime.a
   )

cmake/Modules/testMPIHandshake.cmake

@@ -0,0 +1,28 @@
+MESSAGE(STATUS "Checking whether XIOS supports MPI Handshake")
+
+# Define the test source code (Fortran code) for the MPI Handshake check
+FILE(WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/test_xios_mpi_handshake.f90"
+"
+PROGRAM test_mpi_handshake
+  ! Try importing mpi_handshake modules
+  USE xios, ONLY: xios_mpi_handshake, xios_MAX_GROUPNAME_LEN
+  IMPLICIT NONE
+END PROGRAM test_mpi_handshake
+")
+
+set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -I${XIOS_INCLUDE_DIR}")
+TRY_COMPILE(TEST_XIOS_HAS_MPI_HANDSHAKE
+    ${CMAKE_BINARY_DIR}
+    ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/test_xios_mpi_handshake.f90
+    OUTPUT_VARIABLE OUTPUT
+)
+
+# Check if the compilation was successful
+IF(TEST_XIOS_HAS_MPI_HANDSHAKE)
+  message(STATUS "Checking whether XIOS supports MPI Handshake -- yes")
+  set(XIOS_HAS_MPI_HANDSHAKE 1 CACHE BOOL "")
+ELSE(TEST_XIOS_HAS_MPI_HANDSHAKE)
+  message(STATUS "Checking whether XIOS supports MPI Handshake -- no")
+  set(XIOS_HAS_MPI_HANDSHAKE 0 CACHE BOOL "")
+ENDIF(TEST_XIOS_HAS_MPI_HANDSHAKE)
+MARK_AS_ADVANCED(XIOS_HAS_MPI_HANDSHAKE)

fem/src/CMakeLists.txt

@@ -48,12 +48,6 @@ SET(solverlib_SOURCES AddrFunc.F90 NavierStokes.F90 NavierStokesGeneral.F90
 #SET_PROPERTY(SOURCE MaxwellAxiS.F90 PROPERTY
 #	COMPILE_DEFINITIONS FULL_INDUCTION)
 
-IF(MPI_Fortran_HAVE_F90_MODULE)
-  LIST(APPEND solverlib_SOURCES mo_mpi_handshake.F90 mpi_handshake.c)
-ELSE()
-  LIST(APPEND solverlib_SOURCES mo_mpi_handshake_stub.F90)
-ENDIF()
-
 IF(WITH_YAC)
   LIST(APPEND solverlib_SOURCES elmer_grid.c elmer_coupling.F90)
 ENDIF()

fem/src/Messages.F90

@@ -52,12 +52,11 @@ MODULE Messages
 
 
 #ifdef HAVE_XIOS
-  USE XIOS
+  USE XIOS, ONLY: xios_context_finalize, xios_finalize
 #endif
 
 #ifdef HAVE_YAC
-   USE elmer_coupling
-   USE elmer_icon_coupling
+   USE elmer_coupling, ONLY: coupling_finalize
 #endif
 
    IMPLICIT NONE
@@ -75,13 +74,8 @@ MODULE Messages
 
    INTEGER, PARAMETER :: EXIT_OK=0, EXIT_ERROR=1
 
-#ifdef HAVE_XIOS
    LOGICAL :: USE_XIOS = .FALSE. 
-#endif 
-
-#ifdef HAVE_YAC
    LOGICAL :: USE_YAC = .FALSE.
-#endif   
 
 CONTAINS
 

fem/src/SParIterComm.F90

@@ -49,8 +49,10 @@ MODULE SParIterComm
   USE LoadMod, ONLY : RealTime
   USE SParIterGlobals
 
-#ifdef HAVE_XIOS
-  USE XIOS
+! always use mpi_handshake if YAC is involved; if only XIOS is used, use
+! mpi_handshake if XIOS requires it
+#if defined(HAVE_YAC) || (defined(HAVE_XIOS) && defined(XIOS_HAS_MPI_HANDSHAKE))
+#  define ELMER_USE_MPI_HANDSHAKE
 #endif
 
 #ifndef HAVE_PARMMG
@@ -59,16 +61,43 @@ MODULE SParIterComm
 #  endif
 #endif
 
+#ifdef ELMER_USE_MPI_HANDSHAKE
+  ! do some compatibility checks first
+#  ifdef ELMER_COLOUR
+#    error "It looks like you are trying to use ELMER_COLOUR and mpi_handshake"
+#    error "at the same time. These features are incompatible. Please review"
+#    error "your configuration and dependencies."
+#  endif
+
+#  if defined(HAVE_XIOS) && !defined(XIOS_HAS_MPI_HANDSHAKE)
+#    error "XIOS does not offer the MPI Handshake API (XIOS_HAS_MPI_HANDSHAKE"
+#    error "unset) is not defined, but ELMER_USE_MPI_HANDSHAKE is set."
+#    error "This is incompatible with YAC."
+#  endif
+
+  ! import mpi_handshake from YAC or XIOS
+# ifdef HAVE_XIOS
+    USE XIOS, ONLY: xios_get_global_id
+
+    ! prefer mpi_handshake from XIOS if XIOS is used
+    USE XIOS, ONLY: mpi_handshake => xios_mpi_handshake, &
+                MAX_GROUPNAME_LEN => xios_MAX_GROUPNAME_LEN
+# else
+    ! use mpi_handshake from YAC if only HAVE_YAC used without HAVE_XIOS
+    USE elmer_coupling, ONLY: mpi_handshake, MAX_GROUPNAME_LEN
+# endif
+#endif
+
+#ifdef HAVE_XIOS
+  USE XIOS, ONLY: xios_initialize, xios_context_finalize, xios_finalize
+#endif
+
 #ifdef HAVE_YAC
-  USE elmer_coupling, ONLY: coupling_init, coupling_finalize, coupling_setup, &
-                    mpi_handshake, MAX_GROUPNAME_LEN
-  USE elmer_icon_coupling
-#elif defined(ELMER_HAVE_MPI_MODULE)
-  ! If YAC is not used, use the mpi_handshake from mo_mpi_handshake.F90
-  USE mo_mpi_handshake, ONLY: mpi_handshake, MAX_GROUPNAME_LEN
-#else
-  ! If no MPI is present use a stub
-  USE mo_mpi_handshake_stub, ONLY: mpi_handshake, MAX_GROUPNAME_LEN
+  USE elmer_coupling, ONLY: &
+       coupling_init, &
+       coupling_finalize, &
+       coupling_setup, &
+       coupler_get_code_id
 #endif
 
   IMPLICIT NONE
@@ -94,6 +123,7 @@ MODULE SParIterComm
   INCLUDE "mpif.h"
 #endif
 
+#ifdef ELMER_USE_MPI_HANDSHAKE
   ! Used for MPI_handshake
   ! Classify communicator groups with labels
   INTEGER, PARAMETER :: MAX_NUM_GROUPS = 3
@@ -112,7 +142,7 @@ MODULE SParIterComm
     ! Group for ranks using XIOS, i.e. only Elmer (XIOS clients) + XIOS server
   INTEGER :: XIOS_GROUP_IDX = -1
   CHARACTER(LEN=MAX_GROUPNAME_LEN) :: XIOS_LABEL
-
+#endif
 
   TYPE Buff_t
     REAL(KIND=dp), ALLOCATABLE :: rbuf(:)
@@ -255,16 +285,15 @@ FUNCTION ParCommInit( ) RESULT ( ParallelEnv )
     CALL MPI_COMM_SIZE( MPI_COMM_WORLD, ParEnv % PEs, ierr )
     CALL MPI_COMM_RANK( MPI_COMM_WORLD, ParEnv % MyPE, ierr )
 
-! Use mpi_handshake for comm splitting
-! TODO how to make sure that mpi_handshake does not conflict with MPI_COMM_SPLIT based on ELMER_COLOUR?
-
-! Add Elmer group for comm splitting
-NUM_GROUPS = NUM_GROUPS + 1
-ELMER_GROUP_IDX = NUM_GROUPS
-CALL SetExecID()
-GROUP_NAMES(ELMER_GROUP_IDX) = TRIM(ExecID)
+#ifdef ELMER_USE_MPI_HANDSHAKE
+  ! Use mpi_handshake for comm splitting
+  ! Add Elmer group for comm splitting
+  NUM_GROUPS = NUM_GROUPS + 1
+  ELMER_GROUP_IDX = NUM_GROUPS
+  CALL SetExecID()
+  GROUP_NAMES(ELMER_GROUP_IDX) = TRIM(ExecID)
 
-#ifdef HAVE_XIOS
+#  ifdef HAVE_XIOS
     INQUIRE(FILE="iodef.xml", EXIST=USE_XIOS)
     ! add XIOS group for comm splitting
     IF (USE_XIOS) THEN
@@ -274,71 +303,80 @@ FUNCTION ParCommInit( ) RESULT ( ParallelEnv )
       NUM_GROUPS = NUM_GROUPS + 1
       XIOS_GROUP_IDX = NUM_GROUPS
       GROUP_NAMES(XIOS_GROUP_IDX) = XIOS_LABEL
-    ENDIF
-#endif
+    END IF
+#  endif
 
-#ifdef HAVE_YAC
+#  ifdef HAVE_YAC
     ! check config file and set flag USE_YAC
-    WRITE(config_file,'(A)') "coupling.yaml"
+    WRITE(config_file,*) "coupling.yaml"
     INQUIRE(FILE="coupling.yaml", EXIST=USE_YAC)
     ! add YAC group for comm splitting
     IF (USE_YAC) THEN
-      ! Query mpi_handshake group label from coupler
-      CALL coupler_get_code_id(COUPLER_LABEL)
-      ! Add Group for coupler
-      NUM_GROUPS = NUM_GROUPS + 1
-      COUPLER_GROUP_IDX = NUM_GROUPS
-      GROUP_NAMES(COUPLER_GROUP_IDX) = COUPLER_LABEL
-    ENDIF
-#endif
+        ! Query mpi_handshake group label from coupler
+        CALL coupler_get_code_id(COUPLER_LABEL)
+        ! Add Group for coupler
+        NUM_GROUPS = NUM_GROUPS + 1
+        COUPLER_GROUP_IDX = NUM_GROUPS
+        GROUP_NAMES(COUPLER_GROUP_IDX) = COUPLER_LABEL
+    END IF
+#  endif
 
-IF (NUM_GROUPS > MAX_NUM_GROUPS) THEN
-    WRITE( Message, * ) 'Too many communication groups defined.'
+  IF (NUM_GROUPS > MAX_NUM_GROUPS) THEN
+    WRITE( Message,'(A)') 'Too many communication groups defined.'
     CALL Fatal( 'ParCommInit', Message )
-ENDIF
+  END IF
 
-! Do comm splitting using handshake
-CALL mpi_handshake(MPI_COMM_WORLD, GROUP_NAMES(1:NUM_GROUPS), GROUP_COMMS(1:NUM_GROUPS))
+  IF (USE_XIOS .OR. USE_YAC) THEN
+    CALL mpi_handshake(MPI_COMM_WORLD, GROUP_NAMES(1:NUM_GROUPS),&
+    GROUP_COMMS(1:NUM_GROUPS))
+    ! Set ELMER_COMM_WORLD determined through mpi_handshake
+    ELMER_COMM_WORLD = GROUP_COMMS(ELMER_GROUP_IDX)
+  ELSE
+#endif
 
-ELMER_COMM_WORLD = GROUP_COMMS(ELMER_GROUP_IDX)  ! Set ELMER_COMM_WORLD determined through mpi_handshake
+! The colour could be set to be some different if we want to couple
+! ElmerSolver with some other software having MPI colour set to zero.
+#ifndef ELMER_COLOUR
+#define ELMER_COLOUR 0
+#endif
+
+CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,ELMER_COLOUR,&
+ParEnv % MyPE,ELMER_COMM_WORLD,ierr)
+
+#ifdef ELMER_USE_MPI_HANDSHAKE
+  END IF
+#endif
 
 ! Use XIOS library for IO
 ! Must have xios and iodef.xml present
 #ifdef HAVE_XIOS
-    INQUIRE(FILE="iodef.xml", EXIST=USE_XIOS)
     IF (USE_XIOS) THEN
-      WRITE(Message,*) "Using XIOS with config-file: iodef.xml"
-      CALL INFO("SparIterComm",Message,Level=25)
-      CALL SetExecID()
-      CALL xios_initialize(TRIM(ExecID), global_comm=GROUP_COMMS(XIOS_GROUP_IDX))
-    ELSE
-#ifndef ELMER_COLOUR
-#define ELMER_COLOUR 0
-#endif
-  ! TODO potential incompatibility with MPI_Handshake
-      CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,ELMER_COLOUR,&
-           ParEnv % MyPE,ELMER_COMM_WORLD,ierr) 
-    ENDIF
-#else
-    ! The colour could be set to be some different if we want to couple ElmerSolver with some other
-    ! software having MPI colour set to zero. 
-#ifndef ELMER_COLOUR
-#define ELMER_COLOUR 0
+        WRITE( Message,'(A)') 'Using XIOS with config-file: iodef.xml'
+        CALL INFO("SparIterComm",Message,Level=25)
+        CALL SetExecID()
+#        ifdef ELMER_USE_MPI_HANDSHAKE
+        CALL xios_initialize( &
+              TRIM(ExecID), &
+              global_comm=GROUP_COMMS(XIOS_GROUP_IDX))
+#        else
+        CALL xios_initialize( &
+              TRIM(ExecID), &
+              return_comm=ELMER_COMM_WORLD)
+#        endif
+    END IF
 #endif
-    CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,ELMER_COLOUR,&
-         ParEnv % MyPE,ELMER_COMM_WORLD,ierr) 
-#endif  
 
-! Use YAC library for coupling
-!
 #ifdef HAVE_YAC
-    INQUIRE(FILE="coupling.yaml", EXIST=USE_YAC)
     IF (USE_YAC) THEN
       WRITE(Message,'(A,A)') "Using YAC coupler with config-file:",TRIM(config_file)
       CALL INFO("SparIterComm",Message,Level=25)
-      CALL coupling_init(TRIM(config_file), ELMER_COMM_WORLD, GROUP_COMMS(COUPLER_GROUP_IDX))
+      ! TODO: Refactor to also provide GROUP_NAMES(XIOS_GROUP_IDX) here
+      ! CALL coupling_init("coupling.yaml", ELMER_COMM_WORLD,&
+      ! GROUP_COMMS(COUPLER_GROUP_IDX), GROUP_NAMES(ELMER_GROUP_IDX))
+      CALL coupling_init("coupling.yaml", ELMER_COMM_WORLD,&
+      GROUP_COMMS(COUPLER_GROUP_IDX))
     END IF
-#endif    
+#endif  
 
     ParEnv % ActiveComm = ELMER_COMM_WORLD
 
@@ -932,7 +970,7 @@ SUBROUTINE SParEdgeNumbering( Mesh, Allmesh )
         ELSE
           parentnodes(i,1) =  q
           parentnodes(i,2) =  p
-        ENDIF
+        END IF
 
         ! This is the list of PEs sharing parent node 1:
         list1 => nb(parentnodes(i,1)) % Neighbours
@@ -2210,7 +2248,7 @@ SUBROUTINE SParGlobalNumbering( Mesh, OldMesh, NewNodeCnt, Reorder )
                  parentnodes(i,2) = l 
                  ! This is the list of PEs sharing parent node 2:
                  list2 => Mesh % ParallelInfo % NeighbourList(l) % Neighbours
-              END IF         
+              END IF
            END DO
         END DO
         !
@@ -4382,7 +4420,7 @@ SUBROUTINE BuildRevVecIndices( SplittedMatrix )
       END DO
 !print*,parenv % mype, ' <<<<-----', sproc, realtime()-tt; flush(6)
       DEALLOCATE( Gindices )
-    END IF 
+    END IF
   END DO
 
 !print*,parenv % mype, 'first recv: ', realTime()-tt
@@ -5071,12 +5109,12 @@ SUBROUTINE ParEnvFinalize()
     CALL coupling_finalize()
   END IF
 #endif
-  
+
 #ifdef HAVE_XIOS
   IF (USE_XIOS) THEN
     CALL xios_context_finalize()
     CALL xios_finalize()
-  ENDIF
+  END IF
 #endif
 
   IF (.NOT. ParEnv % ExternalInit) THEN