ComputePmeMgr Class Reference

Inheritance diagram for ComputePmeMgr:

List of all members.

Public Member Functions
	ComputePmeMgr ()
	~ComputePmeMgr ()
void	initialize (CkQdMsg *)
void	initialize_pencils (CkQdMsg *)
void	activate_pencils (CkQdMsg *)
void	recvArrays (CProxy_PmeXPencil, CProxy_PmeYPencil, CProxy_PmeZPencil)
void	sendGrid (void)
void	recvGrid (PmeGridMsg *)
void	gridCalc1 (void)
void	sendTransBarrier (void)
void	sendTrans (void)
void	recvTrans (PmeTransMsg *)
void	gridCalc2 (void)
void	sendUntrans (void)
void	recvUntrans (PmeUntransMsg *)
void	gridCalc3 (void)
void	sendUngrid (void)
void	recvUngrid (PmeGridMsg *)
void	ungridCalc (void)
void	setCompute (ComputePme *c)
int	isPmeProcessor (int p)
Friends
class	ComputePme

---

Constructor & Destructor Documentation

ComputePmeMgr::ComputePmeMgr (   )

Definition at line 330 of file ComputePme.C. : pmeProxy(thisgroup), pmeProxyDir(thisgroup), pmeCompute(0) { CkpvAccess(BOCclass_group).computePmeMgr = thisgroup; #ifdef USE_COMM_LIB ComlibDelegateProxy(&pmeProxy); #endif #ifdef NAMD_FFTW if ( CmiMyRank() == 0 ) { fftw_plan_lock = CmiCreateLock(); } #endif myKSpace = 0; localInfo = new LocalPmeInfo[CkNumPes()]; gridPeMap = new int[CkNumPes()]; transPeMap = new int[CkNumPes()]; recipPeDest = new int[CkNumPes()]; gridPeOrder = new int[CkNumPes()]; transPeOrder = new int[CkNumPes()]; isPmeFlag = new char[CkNumPes()]; kgrid = 0; work = 0; grid_count = 0; trans_count = 0; untrans_count = 0; ungrid_count = 0; gridmsg_reuse= new PmeGridMsg*[CkNumPes()]; useBarrier = 0; sendTransBarrier_received = 0; usePencils = 0; }

ComputePmeMgr::~ComputePmeMgr (   )

Definition at line 976 of file ComputePme.C. { #ifdef NAMD_FFTW if ( CmiMyRank() == 0 ) { CmiDestroyLock(fftw_plan_lock); } #endif delete myKSpace; delete [] localInfo; delete [] gridPeMap; delete [] transPeMap; delete [] recipPeDest; delete [] gridPeOrder; delete [] transPeOrder; delete [] isPmeFlag; delete [] qgrid; if ( kgrid != qgrid ) delete [] kgrid; delete [] work; delete [] gridmsg_reuse; }

---

Member Function Documentation

void ComputePmeMgr::activate_pencils (  CkQdMsg *   )

Definition at line 970 of file ComputePme.C. { if ( ! usePencils ) return; if ( CkMyPe() == 0 ) zPencil.dummyRecvGrid(CkMyPe(),1); }

void ComputePmeMgr::gridCalc1 (  void   )

Definition at line 1043 of file ComputePme.C. References PmeGrid::dim2, and PmeGrid::dim3. { // CkPrintf("gridCalc1 on Pe(%d)\n",CkMyPe()); #ifdef NAMD_FFTW for ( int g=0; g<numGrids; ++g ) { rfftwnd_real_to_complex(forward_plan_yz, localInfo[myGridPe].nx, qgrid + qgrid_size * g, 1, myGrid.dim2 * myGrid.dim3, 0, 0, 0); } #endif #if CHARM_VERSION > 050402 if ( ! useBarrier ) pmeProxyDir[CkMyPe()].sendTrans(); #else if ( ! useBarrier ) pmeProxyDir.sendTrans(CkMyPe()); #endif }

void ComputePmeMgr::gridCalc2 (  void   )

Definition at line 1156 of file ComputePme.C. References BigReal, PmeKSpace::compute_energy(), PmeGrid::dim3, and LocalPmeInfo::ny_after_transpose. { // CkPrintf("gridCalc2 on Pe(%d)\n",CkMyPe()); #if CMK_VERSION_BLUEGENE CmiNetworkProgressAfter (0); #endif int zdim = myGrid.dim3; // int y_start = localInfo[myTransPe].y_start_after_transpose; int ny = localInfo[myTransPe].ny_after_transpose; for ( int g=0; g<numGrids; ++g ) { // finish forward FFT (x dimension) #ifdef NAMD_FFTW fftw(forward_plan_x, ny * zdim / 2, (fftw_complex *)(kgrid+qgrid_size*g), ny * zdim / 2, 1, work, 1, 0); #endif // reciprocal space portion of PME BigReal ewaldcof = ComputeNonbondedUtil::ewaldcof; recip_evir2[g][0] = myKSpace->compute_energy(kgrid+qgrid_size*g, lattice, ewaldcof, &(recip_evir2[g][1])); // CkPrintf("Ewald reciprocal energy = %f\n", recip_evir2[g][0]); // start backward FFT (x dimension) #ifdef NAMD_FFTW fftw(backward_plan_x, ny * zdim / 2, (fftw_complex *)(kgrid+qgrid_size*g), ny * zdim / 2, 1, work, 1, 0); #endif } #if CHARM_VERSION > 050402 pmeProxyDir[CkMyPe()].sendUntrans(); #else pmeProxyDir.sendUntrans(CkMyPe()); #endif }

void ComputePmeMgr::gridCalc3 (  void   )

Definition at line 1291 of file ComputePme.C. References PmeGrid::dim2, and PmeGrid::dim3. { // CkPrintf("gridCalc3 on Pe(%d)\n",CkMyPe()); // finish backward FFT #ifdef NAMD_FFTW for ( int g=0; g<numGrids; ++g ) { rfftwnd_complex_to_real(backward_plan_yz, localInfo[myGridPe].nx, (fftw_complex *) (qgrid + qgrid_size * g), 1, myGrid.dim2 * myGrid.dim3 / 2, 0, 0, 0); } #endif #if CHARM_VERSION > 050402 pmeProxyDir[CkMyPe()].sendUngrid(); #else pmeProxyDir.sendUngrid(CkMyPe()); #endif }

void ComputePmeMgr::initialize (  CkQdMsg *   )

Definition at line 372 of file ComputePme.C. References Lattice::a(), Lattice::a_r(), ResizeArray< Elem >::add(), BigReal, PmeGrid::block1, PmeGrid::block2, PmeGrid::block3, SimParameters::cutoff, SimParameters::decouple, PmeGrid::dim2, PmeGrid::dim3, SimParameters::fepElecLambdaStart, SimParameters::fepOn, SimParameters::FFTWEstimate, generatePmePeList2(), PmePencilInitMsgData::grid, iINFO(), iout, PmeGrid::K1, PmeGrid::K2, PmeGrid::K3, SimParameters::lattice, SimParameters::lesFactor, SimParameters::lesOn, PatchMap::max_a(), PatchMap::min_a(), NAMD_bug(), NAMD_die(), PatchMap::node(), PatchMap::numNodesWithPatches(), PatchMap::numPatches(), PatchMap::numPatchesOnNode(), LocalPmeInfo::nx, LocalPmeInfo::ny_after_transpose, PatchMap::Object(), Node::Object(), PmeGrid::order, SimParameters::pairInteractionOn, SimParameters::pairInteractionSelf, SimParameters::patchDimension, pencilPMEProcessors, SimParameters::PMEBarrier, SimParameters::PMEGridSizeX, SimParameters::PMEGridSizeY, SimParameters::PMEGridSizeZ, SimParameters::PMEInterpOrder, SimParameters::PMEMinPoints, SimParameters::PMEMinSlices, SimParameters::PMEPencils, SimParameters::PMEProcessors, PmePencilInitMsgData::pmeProxy, Random::reorder(), Node::simParameters, simParams, ResizeArray< Elem >::size(), SortableResizeArray< Elem >::sort(), SimParameters::thermInt, Vector::unit(), LocalPmeInfo::x_start, PmePencilInitMsgData::xBlocks, PmePencilInitMsgData::xPencil, LocalPmeInfo::y_start_after_transpose, PmePencilInitMsgData::yBlocks, PmePencilInitMsgData::yPencil, PmePencilInitMsgData::zBlocks, and PmePencilInitMsgData::zPencil. { delete msg; SimParameters *simParams = Node::Object()->simParameters; PatchMap *patchMap = PatchMap::Object(); fepOn = simParams->fepOn; thermInt = simParams->thermInt; decouple = (fepOn || thermInt) && (simParams->decouple); fepElecLambdaStart = (fepOn || thermInt) ? simParams->fepElecLambdaStart : 0; if (fepOn || thermInt) { numGrids = 2; if (decouple) numGrids += 2; if (fepElecLambdaStart || thermInt) numGrids ++; } else numGrids = 1; lesOn = simParams->lesOn; useBarrier = simParams->PMEBarrier; if ( lesOn ) { lesFactor = simParams->lesFactor; numGrids = lesFactor; } selfOn = 0; pairOn = simParams->pairInteractionOn; if ( pairOn ) { selfOn = simParams->pairInteractionSelf; if ( selfOn ) pairOn = 0; // make pairOn and selfOn exclusive numGrids = selfOn ? 1 : 3; } if ( numGrids != 1 || simParams->PMEPencils == 0 ) usePencils = 0; else if ( simParams->PMEPencils > 0 ) usePencils = 1; else { int dimx = simParams->PMEGridSizeX; int dimy = simParams->PMEGridSizeY; int maxslabs = 1 + (dimx - 1) / simParams->PMEMinSlices; if ( maxslabs > CkNumPes() ) maxslabs = CkNumPes(); int maxpencils = ( simParams->PMEGridSizeX * simParams->PMEGridSizeY * simParams->PMEGridSizeZ ) / simParams->PMEMinPoints; if ( maxpencils > CkNumPes() ) maxpencils = CkNumPes(); if ( maxpencils > 3 * maxslabs ) usePencils = 1; else usePencils = 0; } if ( usePencils ) { if ( simParams->PMEPencils > 1 ) { xBlocks = yBlocks = zBlocks = simParams->PMEPencils; } else { int nb2 = ( simParams->PMEGridSizeX * simParams->PMEGridSizeY * simParams->PMEGridSizeZ ) / simParams->PMEMinPoints; if ( nb2 > CkNumPes() ) nb2 = CkNumPes(); int nb = (int) sqrt((float)nb2); xBlocks = zBlocks = nb; yBlocks = nb2 / nb; } int dimx = simParams->PMEGridSizeX; int bx = 1 + ( dimx - 1 ) / xBlocks; xBlocks = 1 + ( dimx - 1 ) / bx; int dimy = simParams->PMEGridSizeY; int by = 1 + ( dimy - 1 ) / yBlocks; yBlocks = 1 + ( dimy - 1 ) / by; int dimz = simParams->PMEGridSizeZ / 2 + 1; // complex int bz = 1 + ( dimz - 1 ) / zBlocks; zBlocks = 1 + ( dimz - 1 ) / bz; if ( ! CkMyPe() ) { iout << iINFO << "PME using " << xBlocks << " x " << yBlocks << " x " << zBlocks << " pencil grid for FFT and reciprocal sum.\n" << endi; } } else { // usePencils { // decide how many pes to use for reciprocal sum // rules based on work available int minslices = simParams->PMEMinSlices; int dimx = simParams->PMEGridSizeX; int nrpx = ( dimx + minslices - 1 ) / minslices; int dimy = simParams->PMEGridSizeY; int nrpy = ( dimy + minslices - 1 ) / minslices; // rules based on processors available int nrpp = CkNumPes(); // if ( nrpp > 32 ) nrpp = 32; // cap to limit messages if ( nrpp < nrpx ) nrpx = nrpp; if ( nrpp < nrpy ) nrpy = nrpp; // user override int nrps = simParams->PMEProcessors; if ( nrps > CkNumPes() ) nrps = CkNumPes(); if ( nrps > 0 ) nrpx = nrps; if ( nrps > 0 ) nrpy = nrps; // make sure there aren't any totally empty processors int bx = ( dimx + nrpx - 1 ) / nrpx; nrpx = ( dimx + bx - 1 ) / bx; int by = ( dimy + nrpy - 1 ) / nrpy; nrpy = ( dimy + by - 1 ) / by; if ( bx != ( dimx + nrpx - 1 ) / nrpx ) NAMD_bug("Error in selecting number of PME processors."); if ( by != ( dimy + nrpy - 1 ) / nrpy ) NAMD_bug("Error in selecting number of PME processors."); numGridPes = nrpx; numTransPes = nrpy; } if ( ! CkMyPe() ) { iout << iINFO << "PME using " << numGridPes << " and " << numTransPes << " processors for FFT and reciprocal sum.\n" << endi; } { // generate random orderings for grid and trans messages int i; for ( i = 0; i < numGridPes; ++i ) { gridPeOrder[i] = i; } for ( i = 0; i < numTransPes; ++i ) { transPeOrder[i] = i; } Random rand(CkMyPe()); rand.reorder(gridPeOrder,numGridPes); rand.reorder(transPeOrder,numTransPes); } int sum_npes = numTransPes + numGridPes; int max_npes = (numTransPes > numGridPes)?numTransPes:numGridPes; #if USE_TOPOMAP PatchMap * pmap = PatchMap::Object(); int patch_pes = pmap->numNodesWithPatches(); TopoManager tmgr; if(tmgr.hasMultipleProcsPerNode()) patch_pes *= 2; bool done = false; #ifndef USE_COMM_LIB if(CkNumPes() > 2*sum_npes + patch_pes) { done = generateBGLORBPmePeList(transPeMap, numTransPes); done &= generateBGLORBPmePeList(gridPeMap, numGridPes, transPeMap, numTransPes); } else #endif if(CkNumPes() > 2 *max_npes + patch_pes) { done = generateBGLORBPmePeList(transPeMap, max_npes); gridPeMap = transPeMap; } if (!done) #endif { //generatePmePeList(transPeMap, max_npes); //gridPeMap = transPeMap; generatePmePeList2(gridPeMap, numGridPes, transPeMap, numTransPes); } #ifdef USE_COMM_LIB if(CkMyPe() == 0) { ComlibInstanceHandle cinst1 = CkCreateComlibInstance(); EachToManyMulticastStrategy *strat = new EachToManyMulticastStrategy(USE_DIRECT, numGridPes, gridPeMap, numTransPes, transPeMap); cinst1.setStrategy(strat); ComlibInstanceHandle cinst2 = CkCreateComlibInstance(); strat = new EachToManyMulticastStrategy(USE_DIRECT, numTransPes, transPeMap , numGridPes, gridPeMap); cinst2.setStrategy(strat); ComlibDoneCreating(); } #endif myGridPe = -1; int i = 0; for ( i=0; i<CkNumPes(); ++i ) isPmeFlag[i] = 0; for ( i=0; i<numGridPes; ++i ) { if ( gridPeMap[i] == CkMyPe() ) myGridPe = i; isPmeFlag[gridPeMap[i]] |= 1; } myTransPe = -1; for ( i=0; i<numTransPes; ++i ) { if ( transPeMap[i] == CkMyPe() ) myTransPe = i; isPmeFlag[transPeMap[i]] |= 2; } if ( ! CkMyPe() ) { iout << iINFO << "PME GRID LOCATIONS:"; int i; for ( i=0; i<numGridPes && i<10; ++i ) { iout << " " << gridPeMap[i]; } if ( i < numGridPes ) iout << " ..."; iout << "\n" << endi; iout << iINFO << "PME TRANS LOCATIONS:"; for ( i=0; i<numTransPes && i<10; ++i ) { iout << " " << transPeMap[i]; } if ( i < numTransPes ) iout << " ..."; iout << "\n" << endi; } } // ! usePencils myGrid.K1 = simParams->PMEGridSizeX; myGrid.K2 = simParams->PMEGridSizeY; myGrid.K3 = simParams->PMEGridSizeZ; myGrid.order = simParams->PMEInterpOrder; myGrid.dim2 = myGrid.K2; myGrid.dim3 = 2 * (myGrid.K3/2 + 1); if ( ! usePencils ) { myGrid.block1 = ( myGrid.K1 + numGridPes - 1 ) / numGridPes; myGrid.block2 = ( myGrid.K2 + numTransPes - 1 ) / numTransPes; myGrid.block3 = myGrid.dim3 / 2; // complex } if ( usePencils ) { myGrid.block1 = ( myGrid.K1 + xBlocks - 1 ) / xBlocks; myGrid.block2 = ( myGrid.K2 + yBlocks - 1 ) / yBlocks; myGrid.block3 = ( myGrid.K3/2 + 1 + zBlocks - 1 ) / zBlocks; // complex if ( CkMyPe() == 0 ) { int basepe = 0; int npe = CkNumPes(); if ( npe > xBlocks*yBlocks && npe > xBlocks*zBlocks && npe > yBlocks*zBlocks ) { // avoid node 0 ++basepe; --npe; } zPencil = CProxy_PmeZPencil::ckNew(); // (xBlocks,yBlocks,1); yPencil = CProxy_PmeYPencil::ckNew(); // (xBlocks,1,zBlocks); xPencil = CProxy_PmeXPencil::ckNew(); // (1,yBlocks,zBlocks); // decide which pes to use by bit reversal and patch use int i; int ncpus = CkNumPes(); // find next highest power of two int npow2 = 1; int nbits = 0; while ( npow2 < ncpus ) { npow2 *= 2; nbits += 1; } // build bit reversal sequence SortableResizeArray<int> patches, nopatches, pmeprocs; PatchMap *pmap = PatchMap::Object(); i = 0; for ( int icpu=0; icpu<ncpus; ++icpu ) { int ri; for ( ri = ncpus; ri >= ncpus; ++i ) { ri = 0; int pow2 = 1; int rpow2 = npow2 / 2; for ( int j=0; j<nbits; ++j ) { ri += rpow2 * ( ( i / pow2 ) % 2 ); pow2 *= 2; rpow2 /= 2; } } // seq[icpu] = ri; if ( ri ) { // keep 0 for special case if ( pmap->numPatchesOnNode(ri) ) patches.add(ri); else nopatches.add(ri); } } // only use zero if it eliminates overloading or has patches int useZero = 0; int npens = xBlocks*yBlocks; if ( npens % ncpus == 0 ) useZero = 1; if ( npens == nopatches.size() + 1 ) useZero = 1; npens += xBlocks*zBlocks; if ( npens % ncpus == 0 ) useZero = 1; if ( npens == nopatches.size() + 1 ) useZero = 1; npens += yBlocks*zBlocks; if ( npens % ncpus == 0 ) useZero = 1; if ( npens == nopatches.size() + 1 ) useZero = 1; // add nopatches then patches in reversed order for ( i=nopatches.size()-1; i>=0; --i ) pmeprocs.add(nopatches[i]); if ( useZero && ! pmap->numPatchesOnNode(0) ) pmeprocs.add(0); for ( i=patches.size()-1; i>=0; --i ) pmeprocs.add(patches[i]); if ( pmap->numPatchesOnNode(0) ) pmeprocs.add(0); int pe = 0; int npes = pmeprocs.size(); SortableResizeArray<int> zprocs(xBlocks*yBlocks); for ( i=0; i<xBlocks*yBlocks; ++i, ++pe ) zprocs[i] = pmeprocs[pe%npes]; zprocs.sort(); SortableResizeArray<int> yprocs(xBlocks*zBlocks); for ( i=0; i<xBlocks*zBlocks; ++i, ++pe ) yprocs[i] = pmeprocs[pe%npes]; yprocs.sort(); SortableResizeArray<int> xprocs(yBlocks*zBlocks); for ( i=0; i<yBlocks*zBlocks; ++i, ++pe ) xprocs[i] = pmeprocs[pe%npes]; xprocs.sort(); pencilPMEProcessors = new char [CkNumPes()]; memset (pencilPMEProcessors, 0, sizeof(char) * CkNumPes()); int x,y,z; iout << iINFO << "PME Z PENCIL LOCATIONS:"; for ( i=0; i<zprocs.size() && i<10; ++i ) { iout << " " << zprocs[i]; } if ( i < zprocs.size() ) iout << " ..."; iout << "\n" << endi; for (pe=0, x = 0; x < xBlocks; ++x) for (y = 0; y < yBlocks; ++y, ++pe ) { zPencil(x,y,0).insert(zprocs[pe]); pencilPMEProcessors[zprocs[pe]] = 1; } zPencil.doneInserting(); iout << iINFO << "PME Y PENCIL LOCATIONS:"; for ( i=0; i<yprocs.size() && i<10; ++i ) { iout << " " << yprocs[i]; } if ( i < yprocs.size() ) iout << " ..."; iout << "\n" << endi; for (pe=0, z = 0; z < zBlocks; ++z ) for (x = 0; x < xBlocks; ++x, ++pe ) { yPencil(x,0,z).insert(yprocs[pe]); pencilPMEProcessors[yprocs[pe]] = 1; } yPencil.doneInserting(); iout << iINFO << "PME X PENCIL LOCATIONS:"; for ( i=0; i<xprocs.size() && i<10; ++i ) { iout << " " << xprocs[i]; } if ( i < xprocs.size() ) iout << " ..."; iout << "\n" << endi; for (pe=0, y = 0; y < yBlocks; ++y ) for (z = 0; z < zBlocks; ++z, ++pe ) { xPencil(0,y,z).insert(xprocs[pe]); pencilPMEProcessors[xprocs[pe]] = 1; } xPencil.doneInserting(); pmeProxy.recvArrays(xPencil,yPencil,zPencil); PmePencilInitMsgData msgdata; msgdata.grid = myGrid; msgdata.xBlocks = xBlocks; msgdata.yBlocks = yBlocks; msgdata.zBlocks = zBlocks; msgdata.xPencil = xPencil; msgdata.yPencil = yPencil; msgdata.zPencil = zPencil; msgdata.pmeProxy = pmeProxyDir; xPencil.init(new PmePencilInitMsg(msgdata)); yPencil.init(new PmePencilInitMsg(msgdata)); zPencil.init(new PmePencilInitMsg(msgdata)); } return; // continue in initialize_pencils() at next startup stage } int pe; int nx = 0; for ( pe = 0; pe < numGridPes; ++pe ) { localInfo[pe].x_start = nx; nx += myGrid.block1; if ( nx > myGrid.K1 ) nx = myGrid.K1; localInfo[pe].nx = nx - localInfo[pe].x_start; } int ny = 0; for ( pe = 0; pe < numTransPes; ++pe ) { localInfo[pe].y_start_after_transpose = ny; ny += myGrid.block2; if ( ny > myGrid.K2 ) ny = myGrid.K2; localInfo[pe].ny_after_transpose = ny - localInfo[pe].y_start_after_transpose; } { // decide how many pes this node exchanges charges with PatchMap *patchMap = PatchMap::Object(); Lattice lattice = simParams->lattice; BigReal sysdima = lattice.a_r().unit() * lattice.a(); BigReal cutoff = simParams->cutoff; BigReal patchdim = simParams->patchDimension; int numPatches = patchMap->numPatches(); int numNodes = CkNumPes(); int *source_flags = new int[numNodes]; int node; for ( node=0; node<numNodes; ++node ) { source_flags[node] = 0; recipPeDest[node] = 0; } // // make sure that we don't get ahead of ourselves on this node // if ( CkMyPe() < numPatches && myRecipPe >= 0 ) { // source_flags[CkMyPe()] = 1; // recipPeDest[myRecipPe] = 1; // } for ( int pid=0; pid < numPatches; ++pid ) { int pnode = patchMap->node(pid); BigReal minx = patchMap->min_a(pid); BigReal maxx = patchMap->max_a(pid); BigReal margina = 0.5 * ( patchdim - cutoff ) / sysdima; // min1 (max1) is smallest (largest) grid line for this patch int min1 = ((int) floor(myGrid.K1 * (minx - margina))) - myGrid.order + 1; int max1 = ((int) floor(myGrid.K1 * (maxx + margina))); for ( int i=min1; i<=max1; ++i ) { int ix = i; while ( ix >= myGrid.K1 ) ix -= myGrid.K1; while ( ix < 0 ) ix += myGrid.K1; // set source_flags[pnode] if this patch sends to our node if ( myGridPe >= 0 && ix >= localInfo[myGridPe].x_start && ix < localInfo[myGridPe].x_start + localInfo[myGridPe].nx ) { source_flags[pnode] = 1; } // set dest_flags[] for node that our patch sends to if ( pnode == CkMyPe() ) { recipPeDest[ix / myGrid.block1] = 1; } } } numSources = 0; numDestRecipPes = 0; for ( node=0; node<numNodes; ++node ) { if ( source_flags[node] ) ++numSources; if ( recipPeDest[node] ) ++numDestRecipPes; } #if 0 if ( numSources ) { iout << iINFO << "PME " << CkMyPe() << " sources:"; for ( node=0; node<numNodes; ++node ) { if ( source_flags[node] ) iout << " " << node; } iout << "\n" << endi; } #endif delete [] source_flags; // CkPrintf("PME on node %d has %d sources and %d destinations\n", // CkMyPe(), numSources, numDestRecipPes); } // decide how many pes this node exchanges charges with (end) ungrid_count = numDestRecipPes; sendTransBarrier_received = 0; if ( myGridPe < 0 && myTransPe < 0 ) return; // the following only for nodes doing reciprocal sum if ( myTransPe >= 0 ) { int k2_start = localInfo[myTransPe].y_start_after_transpose; int k2_end = k2_start + localInfo[myTransPe].ny_after_transpose; myKSpace = new PmeKSpace(myGrid, k2_start, k2_end, 0, myGrid.dim3/2); } int local_size = myGrid.block1 * myGrid.K2 * myGrid.dim3; int local_size_2 = myGrid.block2 * myGrid.K1 * myGrid.dim3; if ( local_size < local_size_2 ) local_size = local_size_2; qgrid = new float[local_size*numGrids]; if ( numGridPes > 1 || numTransPes > 1 ) { kgrid = new float[local_size*numGrids]; } else { kgrid = qgrid; } qgrid_size = local_size; if ( myGridPe >= 0 ) { qgrid_start = localInfo[myGridPe].x_start * myGrid.K2 * myGrid.dim3; qgrid_len = localInfo[myGridPe].nx * myGrid.K2 * myGrid.dim3; fgrid_start = localInfo[myGridPe].x_start * myGrid.K2; fgrid_len = localInfo[myGridPe].nx * myGrid.K2; } int n[3]; n[0] = myGrid.K1; n[1] = myGrid.K2; n[2] = myGrid.K3; #ifdef NAMD_FFTW CmiLock(fftw_plan_lock); work = new fftw_complex[n[0]]; if ( ! CkMyPe() ) iout << iINFO << "Optimizing 4 FFT steps. 1..." << endi; if ( myGridPe >= 0 ) { forward_plan_yz = rfftwnd_create_plan_specific(2, n+1, FFTW_REAL_TO_COMPLEX, ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE ) | FFTW_IN_PLACE | FFTW_USE_WISDOM, qgrid, 1, 0, 0); } if ( ! CkMyPe() ) iout << " 2..." << endi; if ( myTransPe >= 0 ) { forward_plan_x = fftw_create_plan_specific(n[0], FFTW_REAL_TO_COMPLEX, ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE ) | FFTW_IN_PLACE | FFTW_USE_WISDOM, (fftw_complex *) kgrid, localInfo[myTransPe].ny_after_transpose * myGrid.dim3 / 2, work, 1); } if ( ! CkMyPe() ) iout << " 3..." << endi; if ( myTransPe >= 0 ) { backward_plan_x = fftw_create_plan_specific(n[0], FFTW_COMPLEX_TO_REAL, ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE ) | FFTW_IN_PLACE | FFTW_USE_WISDOM, (fftw_complex *) kgrid, localInfo[myTransPe].ny_after_transpose * myGrid.dim3 / 2, work, 1); } if ( ! CkMyPe() ) iout << " 4..." << endi; if ( myGridPe >= 0 ) { backward_plan_yz = rfftwnd_create_plan_specific(2, n+1, FFTW_COMPLEX_TO_REAL, ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE ) | FFTW_IN_PLACE | FFTW_USE_WISDOM, qgrid, 1, 0, 0); } if ( ! CkMyPe() ) iout << " Done.\n" << endi; CmiUnlock(fftw_plan_lock); #else NAMD_die("Sorry, FFTW must be compiled in to use PME."); #endif if ( myGridPe >= 0 && numSources == 0 ) NAMD_bug("PME grid elements exist without sources."); grid_count = numSources; memset( (void*) qgrid, 0, qgrid_size * numGrids * sizeof(float) ); trans_count = numGridPes; }

void ComputePmeMgr::initialize_pencils (  CkQdMsg *   )

Definition at line 901 of file ComputePme.C. References Lattice::a(), Lattice::a_r(), Lattice::b(), Lattice::b_r(), BigReal, PmeGrid::block1, PmeGrid::block2, SimParameters::cutoff, PmeGrid::K1, PmeGrid::K2, SimParameters::lattice, PatchMap::max_a(), PatchMap::max_b(), PatchMap::min_a(), PatchMap::min_b(), PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), Node::Object(), PmeGrid::order, SimParameters::patchDimension, Node::simParameters, simParams, and Vector::unit(). { delete msg; if ( ! usePencils ) return; SimParameters *simParams = Node::Object()->simParameters; PatchMap *patchMap = PatchMap::Object(); Lattice lattice = simParams->lattice; BigReal sysdima = lattice.a_r().unit() * lattice.a(); BigReal sysdimb = lattice.b_r().unit() * lattice.b(); BigReal cutoff = simParams->cutoff; BigReal patchdim = simParams->patchDimension; int numPatches = patchMap->numPatches(); pencilActive = new char[xBlocks*yBlocks]; for ( int i=0; i<xBlocks; ++i ) { for ( int j=0; j<yBlocks; ++j ) { pencilActive[i*yBlocks+j] = 0; } } for ( int pid=0; pid < numPatches; ++pid ) { int pnode = patchMap->node(pid); if ( pnode != CkMyPe() ) continue; BigReal minx = patchMap->min_a(pid); BigReal maxx = patchMap->max_a(pid); BigReal margina = 0.5 * ( patchdim - cutoff ) / sysdima; // min1 (max1) is smallest (largest) grid line for this patch int min1 = ((int) floor(myGrid.K1 * (minx - margina))) - myGrid.order + 1; int max1 = ((int) floor(myGrid.K1 * (maxx + margina))); BigReal miny = patchMap->min_b(pid); BigReal maxy = patchMap->max_b(pid); BigReal marginb = 0.5 * ( patchdim - cutoff ) / sysdimb; // min2 (max2) is smallest (largest) grid line for this patch int min2 = ((int) floor(myGrid.K2 * (miny - marginb))) - myGrid.order + 1; int max2 = ((int) floor(myGrid.K2 * (maxy + marginb))); for ( int i=min1; i<=max1; ++i ) { int ix = i; while ( ix >= myGrid.K1 ) ix -= myGrid.K1; while ( ix < 0 ) ix += myGrid.K1; for ( int j=min2; j<=max2; ++j ) { int jy = j; while ( jy >= myGrid.K2 ) jy -= myGrid.K2; while ( jy < 0 ) jy += myGrid.K2; pencilActive[(ix / myGrid.block1)*yBlocks + (jy / myGrid.block2)] = 1; } } } numPencilsActive = 0; for ( int i=0; i<xBlocks; ++i ) { for ( int j=0; j<yBlocks; ++j ) { if ( pencilActive[i*yBlocks+j] ) { ++numPencilsActive; zPencil(i,j,0).dummyRecvGrid(CkMyPe(),0); } } } //if ( numPencilsActive ) { // CkPrintf("node %d sending to %d pencils\n", CkMyPe(), numPencilsActive); //} ungrid_count = numPencilsActive; }

int ComputePmeMgr::isPmeProcessor (  int  p  )

Definition at line 326 of file ComputePme.C. References pencilPMEProcessors. { return ( usePencils ? pencilPMEProcessors[p] : isPmeFlag[p] ); }

void ComputePmeMgr::recvArrays (  CProxy_PmeXPencil  , CProxy_PmeYPencil  , CProxy_PmeZPencil  )

Definition at line 366 of file ComputePme.C. { xPencil = x; yPencil = y; zPencil = z; }

void ComputePmeMgr::recvGrid (  PmeGridMsg *   )

Definition at line 1002 of file ComputePme.C. References PmeGrid::dim3, PmeGridMsg::fgrid, PmeGridMsg::lattice, NAMD_bug(), PmeGridMsg::qgrid, PmeGridMsg::sequence, PmeGridMsg::zlist, and PmeGridMsg::zlistlen. Referenced by ComputePme::sendPencils(). { // CkPrintf("recvGrid from %d on Pe(%d)\n",msg->sourceNode,CkMyPe()); if ( grid_count == 0 ) { NAMD_bug("Message order failure in ComputePmeMgr::recvGrid\n"); } if ( grid_count == numSources ) { lattice = msg->lattice; sequence = msg->sequence; } int zdim = myGrid.dim3; int zlistlen = msg->zlistlen; int *zlist = msg->zlist; float *qmsg = msg->qgrid; for ( int g=0; g<numGrids; ++g ) { char *f = msg->fgrid + fgrid_len * g; float *q = qgrid + qgrid_size * g; for ( int i=0; i<fgrid_len; ++i ) { if ( f[i] ) { for ( int k=0; k<zlistlen; ++k ) { q[zlist[k]] += *(qmsg++); } } q += zdim; } } gridmsg_reuse[numSources-grid_count] = msg; --grid_count; if ( grid_count == 0 ) { #if CHARM_VERSION > 050402 pmeProxyDir[CkMyPe()].gridCalc1(); if ( useBarrier ) pmeProxyDir[0].sendTransBarrier(); #else pmeProxyDir.gridCalc1(CkMyPe()); if ( useBarrier ) pmeProxyDir.sendTransBarrier(0); #endif } }

void ComputePmeMgr::recvTrans (  PmeTransMsg *   )

Definition at line 1127 of file ComputePme.C. References CmiMemcpy, PmeGrid::dim3, PmeTransMsg::lattice, PmeTransMsg::nx, LocalPmeInfo::ny_after_transpose, PmeTransMsg::qgrid, PmeTransMsg::sequence, and PmeTransMsg::x_start. { // CkPrintf("recvTrans on Pe(%d)\n",CkMyPe()); if ( trans_count == numGridPes ) { lattice = msg->lattice; sequence = msg->sequence; } int zdim = myGrid.dim3; // int y_start = localInfo[myTransPe].y_start_after_transpose; int ny = localInfo[myTransPe].ny_after_transpose; int x_start = msg->x_start; int nx = msg->nx; for ( int g=0; g<numGrids; ++g ) { CmiMemcpy((void*)(kgrid + qgrid_size * g + x_start*ny*zdim), (void*)(msg->qgrid + nx*ny*zdim*g), nx*ny*zdim*sizeof(float)); } delete msg; --trans_count; if ( trans_count == 0 ) { #if CHARM_VERSION > 050402 pmeProxyDir[CkMyPe()].gridCalc2(); #else pmeProxyDir.gridCalc2(CkMyPe()); #endif } }

void ComputePmeMgr::recvUngrid (  PmeGridMsg *   )

Definition at line 1356 of file ComputePme.C. References ComputePme::copyPencils(), ComputePme::copyResults(), and NAMD_bug(). { // CkPrintf("recvUngrid on Pe(%d)\n",CkMyPe()); if ( ungrid_count == 0 ) { NAMD_bug("Message order failure in ComputePmeMgr::recvUngrid\n"); } if ( usePencils ) pmeCompute->copyPencils(msg); else pmeCompute->copyResults(msg); delete msg; --ungrid_count; if ( ungrid_count == 0 ) { #if CHARM_VERSION > 050402 pmeProxyDir[CkMyPe()].ungridCalc(); #else pmeProxyDir.ungridCalc(CkMyPe()); #endif } }

void ComputePmeMgr::recvUntrans (  PmeUntransMsg *   )

Definition at line 1245 of file ComputePme.C. References CmiMemcpy, PmeGrid::dim3, PmeUntransMsg::evir, PmeGrid::K2, LocalPmeInfo::nx, PmeUntransMsg::ny, PmeUntransMsg::qgrid, and PmeUntransMsg::y_start. { // CkPrintf("recvUntrans on Pe(%d)\n",CkMyPe()); if ( untrans_count == numTransPes ) { for ( int g=0; g<numGrids; ++g ) { recip_evir[g] = 0.; } } #if CMK_VERSION_BLUEGENE CmiNetworkProgressAfter (0); #endif int g; for ( g=0; g<numGrids; ++g ) { recip_evir[g] += msg->evir[g]; } int zdim = myGrid.dim3; // int x_start = localInfo[myGridPe].x_start; int nx = localInfo[myGridPe].nx; int y_start = msg->y_start; int ny = msg->ny; int slicelen = myGrid.K2 * zdim; int cpylen = ny * zdim; for ( g=0; g<numGrids; ++g ) { float *q = qgrid + qgrid_size * g + y_start * zdim; float *qmsg = msg->qgrid + nx * cpylen * g; for ( int x = 0; x < nx; ++x ) { CmiMemcpy((void*)q, (void*)qmsg, cpylen*sizeof(float)); q += slicelen; qmsg += cpylen; } } delete msg; --untrans_count; if ( untrans_count == 0 ) { #if CHARM_VERSION > 050402 pmeProxyDir[CkMyPe()].gridCalc3(); #else pmeProxyDir.gridCalc3(CkMyPe()); #endif } }

void ComputePmeMgr::sendGrid (  void   )

Definition at line 998 of file ComputePme.C. References ComputePme::sendData(). { pmeCompute->sendData(numGridPes,gridPeOrder,recipPeDest,gridPeMap); }

void ComputePmeMgr::sendTrans (  void   )

Definition at line 1074 of file ComputePme.C. References CmiMemcpy, PmeGrid::dim3, PmeGrid::K2, PmeTransMsg::lattice, PmeTransMsg::nx, LocalPmeInfo::nx, LocalPmeInfo::ny_after_transpose, PME_TRANS_PRIORITY, PmeTransMsg::qgrid, PmeTransMsg::sequence, SET_PRIORITY, PmeTransMsg::sourceNode, PmeTransMsg::x_start, LocalPmeInfo::x_start, and LocalPmeInfo::y_start_after_transpose. { // CkPrintf("sendTrans on %d\n",myTransPe); // send data for transpose int zdim = myGrid.dim3; int nx = localInfo[myGridPe].nx; int x_start = localInfo[myGridPe].x_start; int slicelen = myGrid.K2 * zdim; #ifdef USE_COMM_LIB ComlibInstanceHandle cinst1 = CkGetComlibInstance(0); cinst1.beginIteration(); #endif #if CMK_VERSION_BLUEGENE CmiNetworkProgressAfter (0); #endif for (int j=0; j<numTransPes; j++) { int pe = transPeOrder[j]; // different order on each node LocalPmeInfo &li = localInfo[pe]; int cpylen = li.ny_after_transpose * zdim; PmeTransMsg *newmsg = new (nx * cpylen * numGrids, PRIORITY_SIZE) PmeTransMsg; newmsg->sourceNode = myGridPe; newmsg->lattice = lattice; newmsg->x_start = x_start; newmsg->nx = nx; for ( int g=0; g<numGrids; ++g ) { float *q = qgrid + qgrid_size * g + li.y_start_after_transpose * zdim; float *qmsg = newmsg->qgrid + nx * cpylen * g; for ( int x = 0; x < nx; ++x ) { CmiMemcpy((void*)qmsg, (void*)q, cpylen*sizeof(float)); q += slicelen; qmsg += cpylen; } } newmsg->sequence = sequence; SET_PRIORITY(newmsg,sequence,PME_TRANS_PRIORITY) #if CHARM_VERSION > 050402 pmeProxy[transPeMap[pe]].recvTrans(newmsg); #else pmeProxy.recvTrans(newmsg,transPeMap[pe]); #endif } untrans_count = numTransPes; #ifdef USE_COMM_LIB cinst1.endIteration(); #endif }

void ComputePmeMgr::sendTransBarrier (  void   )

Definition at line 1060 of file ComputePme.C. { sendTransBarrier_received += 1; // CkPrintf("sendTransBarrier on %d %d\n",myGridPe,numGridPes-sendTransBarrier_received); if ( sendTransBarrier_received < numGridPes ) return; sendTransBarrier_received = 0; for ( int i=0; i<numGridPes; ++i ) { #if CHARM_VERSION > 050402 pmeProxyDir[gridPeMap[i]].sendTrans(); #else pmeProxyDir.sendTrans(gridPeMap[i]); #endif } }

void ComputePmeMgr::sendUngrid (  void   )

Definition at line 1310 of file ComputePme.C. References PmeGrid::dim3, PmeGridMsg::evir, PmeGridMsg::fgrid, PmeGridMsg::len, PME_UNGRID_PRIORITY, PmeGridMsg::qgrid, SET_PRIORITY, PmeGridMsg::sourceNode, PmeGridMsg::start, PmeGridMsg::zlist, and PmeGridMsg::zlistlen. { for ( int j=0; j<numSources; ++j ) { // int msglen = qgrid_len; PmeGridMsg *newmsg = gridmsg_reuse[j]; int pe = newmsg->sourceNode; if ( j == 0 ) { // only need these once for ( int g=0; g<numGrids; ++g ) { newmsg->evir[g] = recip_evir[g]; } } else { for ( int g=0; g<numGrids; ++g ) { newmsg->evir[g] = 0.; } } int zdim = myGrid.dim3; int flen = newmsg->len; int fstart = newmsg->start; int zlistlen = newmsg->zlistlen; int *zlist = newmsg->zlist; float *qmsg = newmsg->qgrid; for ( int g=0; g<numGrids; ++g ) { char *f = newmsg->fgrid + fgrid_len * g; float *q = qgrid + qgrid_size * g + (fstart-fgrid_start) * zdim; for ( int i=0; i<flen; ++i ) { if ( f[i] ) { for ( int k=0; k<zlistlen; ++k ) { *(qmsg++) = q[zlist[k]]; } } q += zdim; } } newmsg->sourceNode = myGridPe; SET_PRIORITY(newmsg,sequence,PME_UNGRID_PRIORITY) #if CHARM_VERSION > 050402 pmeProxyDir[pe].recvUngrid(newmsg); #else pmeProxyDir.recvUngrid(newmsg,pe); #endif } grid_count = numSources; memset( (void*) qgrid, 0, qgrid_size * numGrids * sizeof(float) ); }

void ComputePmeMgr::sendUntrans (  void   )

Definition at line 1194 of file ComputePme.C. References CmiMemcpy, PmeGrid::dim3, PmeUntransMsg::evir, LocalPmeInfo::nx, PmeUntransMsg::ny, LocalPmeInfo::ny_after_transpose, PME_UNTRANS_PRIORITY, PmeUntransMsg::qgrid, SET_PRIORITY, PmeUntransMsg::sourceNode, LocalPmeInfo::x_start, PmeUntransMsg::y_start, and LocalPmeInfo::y_start_after_transpose. { int zdim = myGrid.dim3; int y_start = localInfo[myTransPe].y_start_after_transpose; int ny = localInfo[myTransPe].ny_after_transpose; #ifdef USE_COMM_LIB ComlibInstanceHandle cinst2 = CkGetComlibInstance(1); cinst2.beginIteration(); #endif #if CMK_VERSION_BLUEGENE CmiNetworkProgressAfter (0); #endif // send data for reverse transpose for (int j=0; j<numGridPes; j++) { int pe = gridPeOrder[j]; // different order on each node LocalPmeInfo &li = localInfo[pe]; int x_start =li.x_start; int nx = li.nx; PmeUntransMsg *newmsg = new (nx*ny*zdim*numGrids,numGrids, PRIORITY_SIZE) PmeUntransMsg; newmsg->sourceNode = myTransPe; newmsg->y_start = y_start; newmsg->ny = ny; for ( int g=0; g<numGrids; ++g ) { if ( j == 0 ) { // only need these once newmsg->evir[g] = recip_evir2[g]; } else { newmsg->evir[g] = 0.; } CmiMemcpy((void*)(newmsg->qgrid+nx*ny*zdim*g), (void*)(kgrid + qgrid_size*g + x_start*ny*zdim), nx*ny*zdim*sizeof(float)); } SET_PRIORITY(newmsg,sequence,PME_UNTRANS_PRIORITY) #if CHARM_VERSION > 050402 pmeProxy[gridPeMap[pe]].recvUntrans(newmsg); #else pmeProxy.recvUntrans(newmsg,gridPeMap[pe]); #endif } #ifdef USE_COMM_LIB cinst2.endIteration(); #endif trans_count = numGridPes; }

void ComputePmeMgr::setCompute (  ComputePme *  c  )  [inline]

Definition at line 244 of file ComputePme.C. References ComputePme::setMgr(). { pmeCompute = c; c->setMgr(this); }

void ComputePmeMgr::ungridCalc (  void   )

Definition at line 1376 of file ComputePme.C. References ComputePme::ungridForces(). Referenced by ComputePme::doWork(). { // CkPrintf("ungridCalc on Pe(%d)\n",CkMyPe()); pmeCompute->ungridForces(); ungrid_count = (usePencils ? numPencilsActive : numDestRecipPes ); }

---

Friends And Related Function Documentation

friend class ComputePme [friend]

Definition at line 221 of file ComputePme.C.

---

The documentation for this class was generated from the following file:

• ComputePme.C

---