PmeXPencil Class Reference

Inheritance diagram for PmeXPencil:

Public Member Functions
PmeXPencil_SDAG_CODE	PmeXPencil ()
	PmeXPencil (CkMigrateMessage *)
	~PmeXPencil ()
void	fft_init ()
void	recv_trans (const PmeTransMsg *)
void	forward_fft ()
void	pme_kspace ()
void	backward_fft ()
void	send_untrans ()
void	send_subset_untrans (int fromIdx, int toIdx)
void	node_process_trans (PmeTransMsg *)
void	evir_init ()
Public Member Functions inherited from PmePencil< CBase_PmeXPencil >
	PmePencil ()
	~PmePencil ()
void	base_init (PmePencilInitMsg *msg)
void	order_init (int nBlocks)

Public Attributes
fftw_plan	forward_plan
fftw_plan	backward_plan
int	ny
int	nz
int	recipEvirPe
PmeKSpace *	myKSpace
Public Attributes inherited from PmePencil< CBase_PmeXPencil >
PmePencilInitMsgData	initdata
Lattice	lattice
PmeReduction	evir
int	sequence
AtomicInt	imsg
AtomicInt	imsgb
int	hasData
int	offload
float *	data
float *	work
int *	send_order
int *	needs_reply

Additional Inherited Members
Public Types inherited from PmePencil< CBase_PmeXPencil >
typedef int	AtomicInt

Detailed Description

Definition at line 4788 of file ComputePme.C.

Constructor & Destructor Documentation

PmeXPencil() [1/2]

PmeXPencil_SDAG_CODE PmeXPencil::PmeXPencil ( )

inline

Definition at line 4791 of file ComputePme.C.

References PmePencil< CBase_PmeXPencil >::imsg, PmePencil< CBase_PmeXPencil >::imsgb, myKSpace, and recipEvirPe.

{ __sdag_init();  myKSpace = 0; setMigratable(false); imsg=imsgb=0; recipEvirPe = -999; }

PmeXPencil() [2/2]

PmeXPencil::PmeXPencil ( CkMigrateMessage *  )

inline

Definition at line 4792 of file ComputePme.C.

{ __sdag_init(); }

~PmeXPencil()

PmeXPencil::~PmeXPencil ( )

inline

Definition at line 4793 of file ComputePme.C.

                      {
        #ifdef NAMD_FFTW
        #ifdef NAMD_FFTW_3
                delete [] forward_plans;
                delete [] backward_plans;
        #endif
        #endif
        }

Member Function Documentation

backward_fft()

void PmeXPencil::backward_fft

(

)

Definition at line 5802 of file ComputePme.C.

References backward_plan, CKLOOP_CTRL_PME_BACKWARDFFT, PmePencil< CBase_PmeXPencil >::data, PmePencilInitMsgData::grid, PmePencil< CBase_PmeXPencil >::initdata, PmeGrid::K1, ny, nz, Node::Object(), PmeXZPencilFFT(), Node::simParameters, SimParameters::useCkLoop, PmePencil< CBase_PmeXPencil >::work, PmePencilInitMsgData::yBlocks, and PmePencilInitMsgData::zBlocks.

Referenced by node_process_trans().

                              {
#ifdef NAMD_FFTW
#ifdef MANUAL_DEBUG_FFTW3
  dumpMatrixFloat3("bw_x_b", data, initdata.grid.K1, ny, nz, thisIndex.x, thisIndex.y, thisIndex.z);
#endif

#ifdef NAMD_FFTW_3
#if     CMK_SMP && USE_CKLOOP
  int useCkLoop = Node::Object()->simParameters->useCkLoop;
  if(useCkLoop>=CKLOOP_CTRL_PME_BACKWARDFFT
     && CkNumPes() >= 2 * initdata.yBlocks * initdata.zBlocks) {
          //for(int i=0; i<numPlans; i++) fftwf_execute(backward_plans[i]);
          //transform the above loop
          CkLoop_Parallelize(PmeXZPencilFFT, 1, (void *)backward_plans, CkMyNodeSize(), 0, numPlans-1); //sync
          return;
  }
#endif
  fftwf_execute(backward_plan);
#else
  fftw(backward_plan, ny*nz,
        ((fftw_complex *) data), ny*nz, 1, (fftw_complex *) work, 1, 0);
#endif
#ifdef MANUAL_DEBUG_FFTW3
  dumpMatrixFloat3("bw_x_a", data, initdata.grid.K1, ny, nz, thisIndex.x, thisIndex.y, thisIndex.z);
#endif
#endif
}

evir_init()

void PmeXPencil::evir_init

(

)

Definition at line 4846 of file ComputePme.C.

References findRecipEvirPe(), PmePencil< CBase_PmeXPencil >::initdata, PmePencilInitMsgData::pmeProxy, and recipEvirPe.

                           {
  recipEvirPe = findRecipEvirPe();
  initdata.pmeProxy[recipEvirPe].addRecipEvirClient();
}

fft_init()

void PmeXPencil::fft_init

(

)

Definition at line 5124 of file ComputePme.C.

References backward_plan, PmeGrid::block2, PmeGrid::block3, PmePencil< CBase_PmeXPencil >::data, PmeGrid::dim3, ComputePmeMgr::fftw_plan_lock, fftwf_malloc, forward_plan, PmePencilInitMsgData::grid, PmePencil< CBase_PmeXPencil >::initdata, PmeGrid::K1, PmeGrid::K2, myKSpace, NAMD_die(), ny, nz, PmePencil< CBase_PmeXPencil >::order_init(), PmePencilInitMsgData::pmeNodeProxy, Node::simParameters, simParams, PmePencil< CBase_PmeXPencil >::work, and PmePencilInitMsgData::xBlocks.

                          {
  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
  Node *node = nd.ckLocalBranch();
  SimParameters *simParams = node->simParameters;
#if USE_NODE_PAR_RECEIVE
  ((NodePmeMgr *)CkLocalNodeBranch(initdata.pmeNodeProxy))->registerXPencil(thisIndex,this);
#endif

  int K1 = initdata.grid.K1;
  int K2 = initdata.grid.K2;
  int dim3 = initdata.grid.dim3;
  int block2 = initdata.grid.block2;
  int block3 = initdata.grid.block3;

  ny = block2;
  if ( (thisIndex.y + 1) * block2 > K2 ) ny = K2 - thisIndex.y * block2;
  nz = block3;
  if ( (thisIndex.z+1)*block3 > dim3/2 ) nz = dim3/2 - thisIndex.z*block3;

#ifdef NAMD_FFTW
  CmiLock(ComputePmeMgr::fftw_plan_lock);

  data = (float *) fftwf_malloc( sizeof(float) * K1*ny*nz*2);
  work = new float[2*K1];

  order_init(initdata.xBlocks);

#ifdef NAMD_FFTW_3
  /* need array of sizes for the how many */
  int fftwFlags = simParams->FFTWPatient ? FFTW_PATIENT  : simParams->FFTWEstimate ? FFTW_ESTIMATE  : FFTW_MEASURE ;
  int sizeLines=ny*nz;
  int planLineSizes[1];
  planLineSizes[0]=K1;
  forward_plan = fftwf_plan_many_dft(1, planLineSizes, sizeLines,
                                     (fftwf_complex *) data, NULL, sizeLines, 1,
                                     (fftwf_complex *) data, NULL, sizeLines, 1,
                                   FFTW_FORWARD,
                                     fftwFlags);
  backward_plan = fftwf_plan_many_dft(1, planLineSizes, sizeLines,
                                     (fftwf_complex *) data, NULL, sizeLines, 1,
                                     (fftwf_complex *) data, NULL, sizeLines, 1,
                                          FFTW_BACKWARD,
                                      fftwFlags);

#if     CMK_SMP && USE_CKLOOP
  if(simParams->useCkLoop) {
          //How many FFT plans to be created? The grain-size issue!!.
          //Currently, I am choosing the min(nx, ny) to be coarse-grain
          numPlans = (ny<=nz?ny:nz);
          // limit attempted parallelism due to false sharing
          //if ( numPlans < CkMyNodeSize() ) numPlans = (ny>=nz?ny:nz);
          //if ( numPlans < CkMyNodeSize() ) numPlans = sizeLines;
          if ( sizeLines/numPlans < 4 ) numPlans = 1;
          int howmany = sizeLines/numPlans;
          forward_plans = new fftwf_plan[numPlans];
          backward_plans = new fftwf_plan[numPlans];
          for(int i=0; i<numPlans; i++) {
                  int curStride = i*howmany;              
                  forward_plans[i] = fftwf_plan_many_dft(1, planLineSizes, howmany,
                                                                                                         ((fftwf_complex *)data)+curStride, NULL, sizeLines, 1,
                                                                                                         ((fftwf_complex *)data)+curStride, NULL, sizeLines, 1,
                                                                                                        FFTW_FORWARD,
                                                                                                         fftwFlags);

                  backward_plans[i] = fftwf_plan_many_dft(1, planLineSizes, howmany,
                                                                                                         ((fftwf_complex *)data)+curStride, NULL, sizeLines, 1,
                                                                                                         ((fftwf_complex *)data)+curStride, NULL, sizeLines, 1,
                                                                                                          FFTW_BACKWARD,
                                                                                                         fftwFlags);
          }
  }else
#endif
  {
          forward_plans = NULL;
          backward_plans = NULL;
  }
#else
  forward_plan = fftw_create_plan_specific(K1, FFTW_FORWARD,
        ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE )
        | FFTW_IN_PLACE | FFTW_USE_WISDOM, (fftw_complex *) data,
        ny*nz, (fftw_complex *) work, 1);
  backward_plan = fftw_create_plan_specific(K1, FFTW_BACKWARD,
        ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE )
        | FFTW_IN_PLACE | FFTW_USE_WISDOM, (fftw_complex *) data,
        ny*nz, (fftw_complex *) work, 1);
#endif
  CmiUnlock(ComputePmeMgr::fftw_plan_lock);
#else
  NAMD_die("Sorry, FFTW must be compiled in to use PME.");
#endif

  myKSpace = new PmeKSpace(initdata.grid,
                thisIndex.y*block2, thisIndex.y*block2 + ny,
                thisIndex.z*block3, thisIndex.z*block3 + nz);

}

forward_fft()

void PmeXPencil::forward_fft

(

)

Definition at line 5742 of file ComputePme.C.

References CKLOOP_CTRL_PME_FORWARDFFT, PmePencil< CBase_PmeXPencil >::data, forward_plan, PmePencilInitMsgData::grid, PmePencil< CBase_PmeXPencil >::initdata, PmeGrid::K1, ny, nz, Node::Object(), PmeXZPencilFFT(), Node::simParameters, SimParameters::useCkLoop, PmePencil< CBase_PmeXPencil >::work, PmePencilInitMsgData::yBlocks, and PmePencilInitMsgData::zBlocks.

Referenced by node_process_trans().

                             {
#ifdef NAMD_FFTW

#ifdef MANUAL_DEBUG_FFTW3
  dumpMatrixFloat3("fw_x_b", data, initdata.grid.K1, ny, nz, thisIndex.x, thisIndex.y, thisIndex.z);
#endif

#ifdef NAMD_FFTW_3
#if     CMK_SMP && USE_CKLOOP
  int useCkLoop = Node::Object()->simParameters->useCkLoop;
  if(useCkLoop>=CKLOOP_CTRL_PME_FORWARDFFT
     && CkNumPes() >= 2 * initdata.yBlocks * initdata.zBlocks) {
          //for(int i=0; i<numPlans; i++) fftwf_execute(forward_plans[i]);
          //transform the above loop
          CkLoop_Parallelize(PmeXZPencilFFT, 1, (void *)forward_plans, CkMyNodeSize(), 0, numPlans-1); //sync
          return;
  }
#endif
  fftwf_execute(forward_plan);
#else
  fftw(forward_plan, ny*nz,
        ((fftw_complex *) data), ny*nz, 1, (fftw_complex *) work, 1, 0);
#endif
#ifdef MANUAL_DEBUG_FFTW3
  dumpMatrixFloat3("fw_x_a", data, initdata.grid.K1, ny, nz, thisIndex.x, thisIndex.y, thisIndex.z);
#endif

#endif
}

node_process_trans()

void PmeXPencil::node_process_trans

(

PmeTransMsg *

msg

)

Definition at line 5685 of file ComputePme.C.

References backward_fft(), forward_fft(), PmeTransMsg::hasData, PmePencil< CBase_PmeXPencil >::hasData, PmePencil< CBase_PmeXPencil >::imsg, PmePencil< CBase_PmeXPencil >::initdata, PmePencil< CBase_PmeXPencil >::needs_reply, pme_kspace(), recv_trans(), send_untrans(), PmeTransMsg::sourceNode, and PmePencilInitMsgData::xBlocks.

Referenced by NodePmeMgr::recvXTrans().

{
  if(msg->hasData) hasData=1;
  needs_reply[msg->sourceNode] = msg->hasData;
  recv_trans(msg);
  int limsg;
  CmiMemoryAtomicFetchAndInc(imsg,limsg);
  if(limsg+1 == initdata.xBlocks)
    {
      if(hasData){
        forward_fft();
        pme_kspace();
        backward_fft();
      }
      send_untrans();
      imsg=0;
      CmiMemoryWriteFence();
    }
}

pme_kspace()

void PmeXPencil::pme_kspace

(

)

Definition at line 5772 of file ComputePme.C.

References CKLOOP_CTRL_PME_KSPACE, PmeKSpace::compute_energy(), PmePencil< CBase_PmeXPencil >::data, PmePencil< CBase_PmeXPencil >::evir, ComputeNonbondedUtil::ewaldcof, PmePencil< CBase_PmeXPencil >::initdata, PmePencil< CBase_PmeXPencil >::lattice, myKSpace, Node::Object(), PmePencilInitMsgData::yBlocks, and PmePencilInitMsgData::zBlocks.

Referenced by node_process_trans().

                            {

  evir = 0.;

#ifdef FFTCHECK
  return;
#endif

  BigReal ewaldcof = ComputeNonbondedUtil::ewaldcof;

  int useCkLoop = 0;
#if CMK_SMP && USE_CKLOOP
  if ( Node::Object()->simParameters->useCkLoop >= CKLOOP_CTRL_PME_KSPACE
       && CkNumPes() >= 2 * initdata.yBlocks * initdata.zBlocks ) {
    useCkLoop = 1;
  }
#endif

  // XXX will need to extend pencil decomposition to support LJ-PME
  int numGrids = 1;
  for ( int g=0; g<numGrids; ++g ) {
    evir[0] = myKSpace->compute_energy(data+0*g,
                lattice, ewaldcof, &(evir[1]), useCkLoop);
  }
  
#if USE_NODE_PAR_RECEIVE
    CmiMemoryWriteFence();
#endif
}

recv_trans()

void PmeXPencil::recv_trans

(

const PmeTransMsg *

msg

)

Definition at line 5705 of file ComputePme.C.

References PmeGrid::block1, PmePencil< CBase_PmeXPencil >::data, PmePencilInitMsgData::grid, PmeTransMsg::hasData, PmePencil< CBase_PmeXPencil >::imsg, PmePencil< CBase_PmeXPencil >::initdata, PmeGrid::K1, PmeTransMsg::lattice, PmePencil< CBase_PmeXPencil >::lattice, PmeTransMsg::nx, ny, nz, PmeTransMsg::qgrid, PmeTransMsg::sequence, PmePencil< CBase_PmeXPencil >::sequence, and PmeTransMsg::sourceNode.

Referenced by node_process_trans().

                                                  {
  if ( imsg == 0 ) {
    lattice = msg->lattice;
    sequence = msg->sequence;
  }
  int block1 = initdata.grid.block1;
  int K1 = initdata.grid.K1;
  int ib = msg->sourceNode;
  int nx = msg->nx;
 if ( msg->hasData ) {
  const float *md = msg->qgrid;
  for ( int i=ib*block1; i<(ib*block1+nx); ++i ) {
   float *d = data + i*ny*nz*2;
   for ( int j=0; j<ny; ++j, d += nz*2 ) {
    for ( int k=0; k<nz; ++k ) {
#ifdef ZEROCHECK
      if ( (*md) == 0. ) CkPrintf("0 in YX at %d %d %d %d %d %d %d %d %d\n",
        ib, thisIndex.y, thisIndex.z, i, j, k, nx, ny, nz);
#endif
      d[2*k] = *(md++);
      d[2*k+1] = *(md++);
    }
   }
  }
 } else {
  for ( int i=ib*block1; i<(ib*block1+nx); ++i ) {
   float *d = data + i*ny*nz*2;
   for ( int j=0; j<ny; ++j, d += nz*2 ) {
    for ( int k=0; k<nz; ++k ) {
      d[2*k] = 0;
      d[2*k+1] = 0;
    }
   }
  }
 }
}

send_subset_untrans()

void PmeXPencil::send_subset_untrans	(	int	fromIdx,
		int	toIdx
	)

Definition at line 5835 of file ComputePme.C.

References PmeGrid::block1, PmePencil< CBase_PmeXPencil >::data, PmeUntransMsg::destElem, PmePencilInitMsgData::grid, PmePencil< CBase_PmeXPencil >::initdata, PmeGrid::K1, PmePencil< CBase_PmeXPencil >::needs_reply, PmeUntransMsg::ny, ny, nz, PME_UNTRANS_PRIORITY, PmePencilInitMsgData::pmeNodeProxy, PRIORITY_SIZE, PmeUntransMsg::qgrid, PmePencil< CBase_PmeXPencil >::send_order, PmePencil< CBase_PmeXPencil >::sequence, SET_PRIORITY, PmeUntransMsg::sourceNode, PmePencilInitMsgData::xBlocks, PmePencilInitMsgData::ym, and PmePencilInitMsgData::yPencil.

Referenced by PmeXPencilSendUntrans().

                                                          {
        int xBlocks = initdata.xBlocks;
        int block1 = initdata.grid.block1;      
        int K1 = initdata.grid.K1;

        for(int isend=fromIdx; isend<=toIdx; isend++) {
                int ib = send_order[isend];
                if ( ! needs_reply[ib] ) {
                        PmeAckMsg *msg = new (PRIORITY_SIZE) PmeAckMsg;
                        CmiEnableUrgentSend(1);
                        SET_PRIORITY(msg,sequence,PME_UNTRANS_PRIORITY)
#if USE_NODE_PAR_RECEIVE
                        initdata.yPencil(ib,0,thisIndex.z).recvNodeAck(msg);
#else
                        initdata.yPencil(ib,0,thisIndex.z).recvAck(msg);
#endif
                        CmiEnableUrgentSend(0);
                        continue;
                }
                int nx = block1;
                if ( (ib+1)*block1 > K1 ) nx = K1 - ib*block1;
                PmeUntransMsg *msg = new (nx*ny*nz*2,PRIORITY_SIZE) PmeUntransMsg;
                msg->sourceNode = thisIndex.y;
                msg->ny = ny;
                float *md = msg->qgrid;
                for ( int i=ib*block1; i<(ib*block1+nx); ++i ) {
                        float *d = data + i*ny*nz*2;
                        for ( int j=0; j<ny; ++j, d += nz*2 ) {
                                for ( int k=0; k<nz; ++k ) {
                                        *(md++) = d[2*k];
                                        *(md++) = d[2*k+1];
                                }
                        }
                }
                SET_PRIORITY(msg,sequence,PME_UNTRANS_PRIORITY)
        CmiEnableUrgentSend(1);
#if USE_NODE_PAR_RECEIVE
        msg->destElem=CkArrayIndex3D(ib,0, thisIndex.z);
#if Y_PERSIST 
        CmiUsePersistentHandle(&untrans_handle[isend], 1);
#endif
        initdata.pmeNodeProxy[CmiNodeOf(initdata.ym.ckLocalBranch()->procNum(0,msg->destElem))].recvYUntrans(msg);
#if Y_PERSIST 
        CmiUsePersistentHandle(NULL, 0);
#endif
#else
#if Y_PERSIST 
  //      CmiUsePersistentHandle(&untrans_handle[isend], 1);
#endif
        initdata.yPencil(ib,0,thisIndex.z).recvUntrans(msg);
#if Y_PERSIST 
   //     CmiUsePersistentHandle(NULL, 0);
#endif
#endif
        CmiEnableUrgentSend(0);
        }
}

send_untrans()

void PmeXPencil::send_untrans

(

)

Definition at line 5893 of file ComputePme.C.

References PmeGrid::block1, CKLOOP_CTRL_PME_SENDUNTRANS, PmePencil< CBase_PmeXPencil >::data, PmeUntransMsg::destElem, PmeEvirMsg::evir, PmePencil< CBase_PmeXPencil >::evir, PmePencilInitMsgData::grid, PmePencil< CBase_PmeXPencil >::initdata, PmeGrid::K1, PmePencil< CBase_PmeXPencil >::needs_reply, PmeUntransMsg::ny, ny, nz, Node::Object(), PME_UNGRID_PRIORITY, PME_UNTRANS_PRIORITY, PmePencilInitMsgData::pmeNodeProxy, PmePencilInitMsgData::pmeProxy, PmeXPencilSendUntrans(), PRIORITY_SIZE, PmeUntransMsg::qgrid, recipEvirPe, PmePencil< CBase_PmeXPencil >::send_order, PmePencil< CBase_PmeXPencil >::sequence, SET_PRIORITY, Node::simParameters, PmeUntransMsg::sourceNode, SimParameters::useCkLoop, PmePencilInitMsgData::xBlocks, PmePencilInitMsgData::yBlocks, PmePencilInitMsgData::ym, PmePencilInitMsgData::yPencil, and PmePencilInitMsgData::zBlocks.

Referenced by node_process_trans().

                              {

  { // send energy and virial
    int numGrids = 1;
    PmeEvirMsg *newmsg = new (numGrids, PRIORITY_SIZE) PmeEvirMsg;
    newmsg->evir[0] = evir;
    SET_PRIORITY(newmsg,sequence,PME_UNGRID_PRIORITY)
    CmiEnableUrgentSend(1);
    initdata.pmeProxy[recipEvirPe].recvRecipEvir(newmsg);
    CmiEnableUrgentSend(0);
  }

#if USE_PERSISTENT
  if (untrans_handle == NULL) setup_persistent();
#endif
#if     CMK_SMP && USE_CKLOOP
  int useCkLoop = Node::Object()->simParameters->useCkLoop;
  if(useCkLoop>=CKLOOP_CTRL_PME_SENDUNTRANS
     && CkNumPes() >= 2 * initdata.yBlocks * initdata.zBlocks) {
                int xBlocks = initdata.xBlocks;

#if USE_NODE_PAR_RECEIVE
                //CkLoop_Parallelize(PmeXPencilSendUntrans, 1, (void *)this, CkMyNodeSize(), 0, xBlocks-1, 1); //has to sync
                CkLoop_Parallelize(PmeXPencilSendUntrans, 1, (void *)this, xBlocks, 0, xBlocks-1, 1); //has to sync
#else
        //CkLoop_Parallelize(PmeXPencilSendUntrans, 1, (void *)this, CkMyNodeSize(), 0, xBlocks-1, 0); //not sync
                CkLoop_Parallelize(PmeXPencilSendUntrans, 1, (void *)this, xBlocks, 0, xBlocks-1, 0); //not sync
#endif        
                return;
  }
#endif
  int xBlocks = initdata.xBlocks;
  int block1 = initdata.grid.block1;
  int K1 = initdata.grid.K1;
  for ( int isend=0; isend<xBlocks; ++isend ) {
    int ib = send_order[isend];
    if ( ! needs_reply[ib] ) {
      PmeAckMsg *msg = new (PRIORITY_SIZE) PmeAckMsg;
      CmiEnableUrgentSend(1);
      SET_PRIORITY(msg,sequence,PME_UNTRANS_PRIORITY)
#if USE_NODE_PAR_RECEIVE
      initdata.yPencil(ib,0,thisIndex.z).recvNodeAck(msg);
#else
      initdata.yPencil(ib,0,thisIndex.z).recvAck(msg);
#endif
      CmiEnableUrgentSend(0);
      continue;
    }
    int nx = block1;
    if ( (ib+1)*block1 > K1 ) nx = K1 - ib*block1;
    PmeUntransMsg *msg = new (nx*ny*nz*2,PRIORITY_SIZE) PmeUntransMsg;
    msg->sourceNode = thisIndex.y;
    msg->ny = ny;
    float *md = msg->qgrid;
    for ( int i=ib*block1; i<(ib*block1+nx); ++i ) {
     float *d = data + i*ny*nz*2;
     for ( int j=0; j<ny; ++j, d += nz*2 ) {
      for ( int k=0; k<nz; ++k ) {
        *(md++) = d[2*k];
        *(md++) = d[2*k+1];
      }
     }
    }
    SET_PRIORITY(msg,sequence,PME_UNTRANS_PRIORITY)

    CmiEnableUrgentSend(1);
#if USE_NODE_PAR_RECEIVE
    msg->destElem=CkArrayIndex3D(ib,0, thisIndex.z);
#if Y_PERSIST 
    CmiUsePersistentHandle(&untrans_handle[isend], 1);
#endif
    initdata.pmeNodeProxy[CmiNodeOf(initdata.ym.ckLocalBranch()->procNum(0,msg->destElem))].recvYUntrans(msg);
#if Y_PERSIST 
    CmiUsePersistentHandle(NULL, 0);
#endif
#else
#if Y_PERSIST 
    CmiUsePersistentHandle(&untrans_handle[isend], 1);
#endif
    initdata.yPencil(ib,0,thisIndex.z).recvUntrans(msg);
#if Y_PERSIST 
    CmiUsePersistentHandle(NULL, 0);
#endif
#endif
    CmiEnableUrgentSend(0);
  }
}

Member Data Documentation

backward_plan

fftw_plan PmeXPencil::backward_plan

Definition at line 4816 of file ComputePme.C.

Referenced by backward_fft(), and fft_init().

forward_plan

fftw_plan PmeXPencil::forward_plan

Definition at line 4816 of file ComputePme.C.

Referenced by fft_init(), and forward_fft().

myKSpace

PmeKSpace* PmeXPencil::myKSpace

Definition at line 4822 of file ComputePme.C.

Referenced by fft_init(), pme_kspace(), and PmeXPencil().

ny

int PmeXPencil::ny

Definition at line 4819 of file ComputePme.C.

Referenced by backward_fft(), fft_init(), forward_fft(), recv_trans(), send_subset_untrans(), and send_untrans().

nz

int PmeXPencil::nz

Definition at line 4819 of file ComputePme.C.

Referenced by backward_fft(), fft_init(), forward_fft(), recv_trans(), send_subset_untrans(), and send_untrans().

recipEvirPe

int PmeXPencil::recipEvirPe

Definition at line 4820 of file ComputePme.C.

Referenced by evir_init(), PmeXPencil(), and send_untrans().

---

The documentation for this class was generated from the following file:

• ComputePme.C