Current Interpolation: Update Event System

Reorder dependencies in the event system and
add a GUARD receive functionality to FieldJ.

Author: ax3l

src/picongpu/include/fields/FieldJ.hpp

@@ -124,6 +124,7 @@ class FieldJ : public SimulationFieldHelper<MappingDesc>, public ISimulationData
 private:
 
     GridBuffer<ValueType, simDim> fieldJ;
+    GridBuffer<ValueType, simDim>* fieldJrecv;
 
     FieldE *fieldE;
     FieldB *fieldB;

src/picongpu/include/fields/FieldJ.tpp

@@ -53,7 +53,7 @@ using namespace PMacc;
 
 FieldJ::FieldJ( MappingDesc cellDescription ) :
 SimulationFieldHelper<MappingDesc>( cellDescription ),
-fieldJ( cellDescription.getGridLayout( ) ), fieldE( NULL ), fieldB( NULL )
+fieldJ( cellDescription.getGridLayout( ) ), fieldE( NULL ), fieldB( NULL ), fieldJrecv( NULL )
 {
     const DataSpace<simDim> coreBorderSize = cellDescription.getGridLayout( ).getDataSpaceWithoutGuarding( );
 
@@ -115,10 +115,34 @@ fieldJ( cellDescription.getGridLayout( ) ), fieldE( NULL ), fieldB( NULL )
         // std::cout << "ex " << i << " x=" << guardingCells[0] << " y=" << guardingCells[1] << " z=" << guardingCells[2] << std::endl;
         fieldJ.addExchangeBuffer( i, guardingCells, FIELD_J );
     }
+
+    /* Receive border values in own guard for "receive" communication pattern - necessary for current interpolation/filter */
+    const DataSpace<simDim> originRecvGuard( GetMargin<fieldSolver::CurrentInterpolation>::LowerMargin( ).toRT( ) );
+    const DataSpace<simDim> endRecvGuard( GetMargin<fieldSolver::CurrentInterpolation>::UpperMargin( ).toRT( ) );
+    if( originRecvGuard != DataSpace<simDim>::create(0) ||
+        endRecvGuard != DataSpace<simDim>::create(0) )
+    {
+        fieldJrecv = new GridBuffer<ValueType, simDim > ( fieldJ.getDeviceBuffer(), cellDescription.getGridLayout( ) );
+
+        /*go over all directions*/
+        for ( uint32_t i = 1; i < NumberOfExchanges<simDim>::value; ++i )
+        {
+            DataSpace<simDim> relativMask = Mask::getRelativeDirections<simDim > ( i );
+            /* guarding cells depend on direction
+             * for negative direction use originGuard else endGuard (relative direction ZERO is ignored)
+             * don't switch end and origin because this is a read buffer and no send buffer
+             */
+            DataSpace<simDim> guardingCells;
+            for ( uint32_t d = 0; d < simDim; ++d )
+                guardingCells[d] = ( relativMask[d] == -1 ? originRecvGuard[d] : endRecvGuard[d] );
+            fieldJrecv->addExchange( GUARD, i, guardingCells, FIELD_JRECV );
+        }
+    }
 }
 
 FieldJ::~FieldJ( )
 {
+    __delete(fieldJrecv);
 }
 
 SimulationDataId FieldJ::getUniqueId( )
@@ -146,7 +170,14 @@ EventTask FieldJ::asyncCommunication( EventTask serialEvent )
     __startTransaction( serialEvent );
     FieldFactory::getInstance( ).createTaskFieldSend( *this );
     ret += __endTransaction( );
-    return ret;
+
+    if( fieldJrecv != NULL )
+    {
+        EventTask eJ = fieldJrecv->asyncCommunication( ret );
+        return eJ;
+    }
+    else
+        return ret;
 }
 
 void FieldJ::bashField( uint32_t exchangeType )

src/picongpu/include/simulationControl/MySimulation.hpp

@@ -415,13 +415,33 @@ class MySimulation : public SimulationHelper<simDim>
 #endif
 
 #if  (ENABLE_CURRENT == 1)
-        if(bmpl::size<VectorAllSpecies>::type::value>0)
+        if(bmpl::size<VectorAllSpecies>::type::value > 0)
         {
             EventTask eRecvCurrent = fieldJ->asyncCommunication(__getTransactionEvent());
-            fieldJ->addCurrentToEMF<CORE >(*myCurrentInterpolation);
 
-            __setTransactionEvent(eRecvCurrent);
-            fieldJ->addCurrentToEMF<BORDER >(*myCurrentInterpolation);
+            const DataSpace<simDim> currentRecvLower( GetMargin<fieldSolver::CurrentInterpolation>::LowerMargin( ).toRT( ) );
+            const DataSpace<simDim> currentRecvUpper( GetMargin<fieldSolver::CurrentInterpolation>::UpperMargin( ).toRT( ) );
+
+            /* without interpolation, we do not need to access the FieldJ GUARD
+             * and can therefor overlap communication of GUARD->(ADD)BORDER & computation of CORE */
+            if( currentRecvLower == DataSpace<simDim>::create(0) &&
+                currentRecvUpper == DataSpace<simDim>::create(0) )
+            {
+                fieldJ->addCurrentToEMF<CORE >(*myCurrentInterpolation);
+                __setTransactionEvent(eRecvCurrent);
+                fieldJ->addCurrentToEMF<BORDER >(*myCurrentInterpolation);
+            } else
+            {
+                /* in case we perform a current interpolation/filter, we need
+                 * to access the BORDER area from the CORE (and the GUARD area
+                 * from the BORDER)
+                 * `fieldJ->asyncCommunication` first adds the neighbors' values
+                 * to BORDER (send) and then updates the GUARD (receive)
+                 * \todo split the last `receive` part in a separate method to
+                 *       allow already a computation of CORE */
+                __setTransactionEvent(eRecvCurrent);
+                fieldJ->addCurrentToEMF<CORE + BORDER>(*myCurrentInterpolation);
+            }
         }
 #endif
 

src/picongpu/include/simulation_types.hpp

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013 Axel Huebl, Felix Schmitt, Heiko Burau, Rene Widera
+ * Copyright 2013-2015 Axel Huebl, Felix Schmitt, Heiko Burau, Rene Widera
  *
  * This file is part of PIConGPU.
  *
@@ -19,7 +19,6 @@
  */
 
 
-
 #pragma once
 
 #include "version.hpp"
@@ -48,8 +47,8 @@ enum ParticleType
 
 enum CommunicationTag
 {
-    FIELD_B = 0u, FIELD_E = 1u, FIELD_J = 2u, FIELD_TMP = 3u,
-    PAR_IONS = 4u, PAR_ELECTRONS = 5u,
+    FIELD_B = 0u, FIELD_E = 1u, FIELD_J = 2u, FIELD_JRECV = 3u, FIELD_TMP = 4u,
+    PAR_IONS = 5u, PAR_ELECTRONS = 6u,
     NO_COMMUNICATION = 16u
 };