Inheritance diagram for ComputePmeMgr:

Public Member Functions | |
| ComputePmeMgr () | |
| ~ComputePmeMgr () | |
| void | initialize (CkQdMsg *) |
| void | initialize_pencils (CkQdMsg *) |
| void | activate_pencils (CkQdMsg *) |
| void | recvArrays (CProxy_PmeXPencil, CProxy_PmeYPencil, CProxy_PmeZPencil) |
| void | sendGrid (void) |
| void | recvGrid (PmeGridMsg *) |
| void | gridCalc1 (void) |
| void | sendTransBarrier (void) |
| void | sendTrans (void) |
| void | recvTrans (PmeTransMsg *) |
| void | gridCalc2 (void) |
| void | sendUntrans (void) |
| void | recvUntrans (PmeUntransMsg *) |
| void | gridCalc3 (void) |
| void | sendUngrid (void) |
| void | recvUngrid (PmeGridMsg *) |
| void | ungridCalc (void) |
| void | setCompute (ComputePme *c) |
| int | isPmeProcessor (int p) |
Static Public Attributes | |
| CmiNodeLock | fftw_plan_lock |
Friends | |
| class | ComputePme |
|
|
Definition at line 333 of file ComputePme.C. References fftw_plan_lock. 00333 : pmeProxy(thisgroup),
00334 pmeProxyDir(thisgroup), pmeCompute(0) {
00335
00336 CkpvAccess(BOCclass_group).computePmeMgr = thisgroup;
00337
00338 #ifdef USE_COMM_LIB
00339 ComlibDelegateProxy(&pmeProxy);
00340 #endif
00341
00342 #ifdef NAMD_FFTW
00343 if ( CmiMyRank() == 0 ) {
00344 fftw_plan_lock = CmiCreateLock();
00345 }
00346 #endif
00347
00348 myKSpace = 0;
00349 kgrid = 0;
00350 work = 0;
00351 grid_count = 0;
00352 trans_count = 0;
00353 untrans_count = 0;
00354 ungrid_count = 0;
00355 gridmsg_reuse= new PmeGridMsg*[CkNumPes()];
00356 useBarrier = 0;
00357 sendTransBarrier_received = 0;
00358 usePencils = 0;
00359 }
|
|
|
Definition at line 981 of file ComputePme.C. References fftw_plan_lock. 00981 {
00982
00983 #ifdef NAMD_FFTW
00984 if ( CmiMyRank() == 0 ) {
00985 CmiDestroyLock(fftw_plan_lock);
00986 }
00987 #endif
00988
00989 delete myKSpace;
00990 delete [] localInfo;
00991 delete [] gridPeMap;
00992 delete [] transPeMap;
00993 delete [] recipPeDest;
00994 delete [] gridPeOrder;
00995 delete [] transPeOrder;
00996 delete [] isPmeFlag;
00997 delete [] qgrid;
00998 if ( kgrid != qgrid ) delete [] kgrid;
00999 delete [] work;
01000 delete [] gridmsg_reuse;
01001 }
|
|
|
Definition at line 975 of file ComputePme.C. 00975 {
00976 if ( ! usePencils ) return;
00977 if ( CkMyPe() == 0 ) zPencil.dummyRecvGrid(CkMyPe(),1);
00978 }
|
|
|
Definition at line 1043 of file ComputePme.C. References PmeGrid::dim2, and PmeGrid::dim3. 01043 {
01044 // CkPrintf("gridCalc1 on Pe(%d)\n",CkMyPe());
01045
01046 #ifdef NAMD_FFTW
01047 for ( int g=0; g<numGrids; ++g ) {
01048 rfftwnd_real_to_complex(forward_plan_yz, localInfo[myGridPe].nx,
01049 qgrid + qgrid_size * g, 1, myGrid.dim2 * myGrid.dim3, 0, 0, 0);
01050 }
01051 #endif
01052
01053 if ( ! useBarrier ) pmeProxyDir[CkMyPe()].sendTrans();
01054 }
|
|
|
Definition at line 1140 of file ComputePme.C. References BigReal, PmeKSpace::compute_energy(), PmeGrid::dim3, and LocalPmeInfo::ny_after_transpose. 01140 {
01141 // CkPrintf("gridCalc2 on Pe(%d)\n",CkMyPe());
01142
01143 #if CMK_BLUEGENEL
01144 CmiNetworkProgressAfter (0);
01145 #endif
01146
01147 int zdim = myGrid.dim3;
01148 // int y_start = localInfo[myTransPe].y_start_after_transpose;
01149 int ny = localInfo[myTransPe].ny_after_transpose;
01150
01151 for ( int g=0; g<numGrids; ++g ) {
01152 // finish forward FFT (x dimension)
01153 #ifdef NAMD_FFTW
01154 fftw(forward_plan_x, ny * zdim / 2, (fftw_complex *)(kgrid+qgrid_size*g),
01155 ny * zdim / 2, 1, work, 1, 0);
01156 #endif
01157
01158 // reciprocal space portion of PME
01159 BigReal ewaldcof = ComputeNonbondedUtil::ewaldcof;
01160 recip_evir2[g][0] = myKSpace->compute_energy(kgrid+qgrid_size*g,
01161 lattice, ewaldcof, &(recip_evir2[g][1]));
01162 // CkPrintf("Ewald reciprocal energy = %f\n", recip_evir2[g][0]);
01163
01164 // start backward FFT (x dimension)
01165 #ifdef NAMD_FFTW
01166 fftw(backward_plan_x, ny * zdim / 2, (fftw_complex *)(kgrid+qgrid_size*g),
01167 ny * zdim / 2, 1, work, 1, 0);
01168 #endif
01169 }
01170
01171 pmeProxyDir[CkMyPe()].sendUntrans();
01172 }
|
|
|
Definition at line 1263 of file ComputePme.C. References PmeGrid::dim2, and PmeGrid::dim3. 01263 {
01264 // CkPrintf("gridCalc3 on Pe(%d)\n",CkMyPe());
01265
01266 // finish backward FFT
01267 #ifdef NAMD_FFTW
01268 for ( int g=0; g<numGrids; ++g ) {
01269 rfftwnd_complex_to_real(backward_plan_yz, localInfo[myGridPe].nx,
01270 (fftw_complex *) (qgrid + qgrid_size * g),
01271 1, myGrid.dim2 * myGrid.dim3 / 2, 0, 0, 0);
01272 }
01273 #endif
01274
01275 pmeProxyDir[CkMyPe()].sendUngrid();
01276 }
|
|
|
Definition at line 368 of file ComputePme.C. References Lattice::a(), Lattice::a_r(), ResizeArray< Elem >::add(), SimParameters::alchDecouple, SimParameters::alchElecLambdaStart, SimParameters::alchFepOn, SimParameters::alchThermIntOn, BigReal, PmeGrid::block1, PmeGrid::block2, PmeGrid::block3, SimParameters::cutoff, PmeGrid::dim2, PmeGrid::dim3, fftw_plan_lock, SimParameters::FFTWEstimate, generatePmePeList2(), PmePencilInitMsgData::grid, iINFO(), ResizeArray< Elem >::insert(), iout, j, PmeGrid::K1, PmeGrid::K2, PmeGrid::K3, SimParameters::lattice, SimParameters::lesFactor, SimParameters::lesOn, PatchMap::max_a(), PatchMap::min_a(), NAMD_bug(), NAMD_die(), PatchMap::node(), PatchMap::numNodesWithPatches(), PatchMap::numPatches(), PatchMap::numPatchesOnNode(), LocalPmeInfo::nx, LocalPmeInfo::ny_after_transpose, PatchMap::Object(), Node::Object(), PmeGrid::order, SimParameters::pairInteractionOn, SimParameters::pairInteractionSelf, SimParameters::patchDimension, pencilPMEProcessors, SimParameters::PMEBarrier, SimParameters::PMEGridSizeX, SimParameters::PMEGridSizeY, SimParameters::PMEGridSizeZ, SimParameters::PMEInterpOrder, SimParameters::PMEMinPoints, SimParameters::PMEMinSlices, SimParameters::PMEPencils, SimParameters::PMEProcessors, PmePencilInitMsgData::pmeProxy, Random::reorder(), Node::simParameters, simParams, ResizeArray< Elem >::size(), SortableResizeArray< Elem >::sort(), Vector::unit(), LocalPmeInfo::x_start, PmePencilInitMsgData::xBlocks, PmePencilInitMsgData::xPencil, LocalPmeInfo::y_start_after_transpose, PmePencilInitMsgData::yBlocks, PmePencilInitMsgData::yPencil, PmePencilInitMsgData::zBlocks, and PmePencilInitMsgData::zPencil. 00368 {
00369 delete msg;
00370
00371 localInfo = new LocalPmeInfo[CkNumPes()];
00372 gridPeMap = new int[CkNumPes()];
00373 transPeMap = new int[CkNumPes()];
00374 recipPeDest = new int[CkNumPes()];
00375 gridPeOrder = new int[CkNumPes()];
00376 transPeOrder = new int[CkNumPes()];
00377 isPmeFlag = new char[CkNumPes()];
00378
00379 SimParameters *simParams = Node::Object()->simParameters;
00380 PatchMap *patchMap = PatchMap::Object();
00381
00382 alchFepOn = simParams->alchFepOn;
00383 alchThermIntOn = simParams->alchThermIntOn;
00384 alchDecouple = (alchFepOn || alchThermIntOn) && (simParams->alchDecouple);
00385 alchElecLambdaStart = (alchFepOn || alchThermIntOn) ?
00386 simParams->alchElecLambdaStart : 0;
00387 if (alchFepOn || alchThermIntOn) {
00388 numGrids = 2;
00389 if (alchDecouple) numGrids += 2;
00390 if (alchElecLambdaStart || alchThermIntOn) numGrids ++;
00391 }
00392 else numGrids = 1;
00393 lesOn = simParams->lesOn;
00394 useBarrier = simParams->PMEBarrier;
00395 if ( lesOn ) {
00396 lesFactor = simParams->lesFactor;
00397 numGrids = lesFactor;
00398 }
00399 selfOn = 0;
00400 pairOn = simParams->pairInteractionOn;
00401 if ( pairOn ) {
00402 selfOn = simParams->pairInteractionSelf;
00403 if ( selfOn ) pairOn = 0; // make pairOn and selfOn exclusive
00404 numGrids = selfOn ? 1 : 3;
00405 }
00406
00407 if ( numGrids != 1 || simParams->PMEPencils == 0 ) usePencils = 0;
00408 else if ( simParams->PMEPencils > 0 ) usePencils = 1;
00409 else {
00410 int dimx = simParams->PMEGridSizeX;
00411 int dimy = simParams->PMEGridSizeY;
00412 int maxslabs = 1 + (dimx - 1) / simParams->PMEMinSlices;
00413 if ( maxslabs > CkNumPes() ) maxslabs = CkNumPes();
00414 int maxpencils = ( simParams->PMEGridSizeX * simParams->PMEGridSizeY
00415 * simParams->PMEGridSizeZ ) / simParams->PMEMinPoints;
00416 if ( maxpencils > CkNumPes() ) maxpencils = CkNumPes();
00417 if ( maxpencils > 3 * maxslabs ) usePencils = 1;
00418 else usePencils = 0;
00419 }
00420
00421 if ( usePencils ) {
00422 if ( simParams->PMEPencils > 1 ) {
00423 xBlocks = yBlocks = zBlocks = simParams->PMEPencils;
00424 } else {
00425 int nb2 = ( simParams->PMEGridSizeX * simParams->PMEGridSizeY
00426 * simParams->PMEGridSizeZ ) / simParams->PMEMinPoints;
00427 if ( nb2 > CkNumPes() ) nb2 = CkNumPes();
00428 int nb = (int) sqrt((float)nb2);
00429 xBlocks = zBlocks = nb;
00430 yBlocks = nb2 / nb;
00431 }
00432
00433 int dimx = simParams->PMEGridSizeX;
00434 int bx = 1 + ( dimx - 1 ) / xBlocks;
00435 xBlocks = 1 + ( dimx - 1 ) / bx;
00436
00437 int dimy = simParams->PMEGridSizeY;
00438 int by = 1 + ( dimy - 1 ) / yBlocks;
00439 yBlocks = 1 + ( dimy - 1 ) / by;
00440
00441 int dimz = simParams->PMEGridSizeZ / 2 + 1; // complex
00442 int bz = 1 + ( dimz - 1 ) / zBlocks;
00443 zBlocks = 1 + ( dimz - 1 ) / bz;
00444
00445 if ( ! CkMyPe() ) {
00446 iout << iINFO << "PME using " << xBlocks << " x " <<
00447 yBlocks << " x " << zBlocks <<
00448 " pencil grid for FFT and reciprocal sum.\n" << endi;
00449 }
00450 } else { // usePencils
00451
00452 { // decide how many pes to use for reciprocal sum
00453
00454 // rules based on work available
00455 int minslices = simParams->PMEMinSlices;
00456 int dimx = simParams->PMEGridSizeX;
00457 int nrpx = ( dimx + minslices - 1 ) / minslices;
00458 int dimy = simParams->PMEGridSizeY;
00459 int nrpy = ( dimy + minslices - 1 ) / minslices;
00460
00461 // rules based on processors available
00462 int nrpp = CkNumPes();
00463 // if ( nrpp > 32 ) nrpp = 32; // cap to limit messages
00464 if ( nrpp < nrpx ) nrpx = nrpp;
00465 if ( nrpp < nrpy ) nrpy = nrpp;
00466
00467 // user override
00468 int nrps = simParams->PMEProcessors;
00469 if ( nrps > CkNumPes() ) nrps = CkNumPes();
00470 if ( nrps > 0 ) nrpx = nrps;
00471 if ( nrps > 0 ) nrpy = nrps;
00472
00473 // make sure there aren't any totally empty processors
00474 int bx = ( dimx + nrpx - 1 ) / nrpx;
00475 nrpx = ( dimx + bx - 1 ) / bx;
00476 int by = ( dimy + nrpy - 1 ) / nrpy;
00477 nrpy = ( dimy + by - 1 ) / by;
00478 if ( bx != ( dimx + nrpx - 1 ) / nrpx )
00479 NAMD_bug("Error in selecting number of PME processors.");
00480 if ( by != ( dimy + nrpy - 1 ) / nrpy )
00481 NAMD_bug("Error in selecting number of PME processors.");
00482
00483 numGridPes = nrpx;
00484 numTransPes = nrpy;
00485 }
00486 if ( ! CkMyPe() ) {
00487 iout << iINFO << "PME using " << numGridPes << " and " << numTransPes <<
00488 " processors for FFT and reciprocal sum.\n" << endi;
00489 }
00490 { // generate random orderings for grid and trans messages
00491 int i;
00492 for ( i = 0; i < numGridPes; ++i ) {
00493 gridPeOrder[i] = i;
00494 }
00495 for ( i = 0; i < numTransPes; ++i ) {
00496 transPeOrder[i] = i;
00497 }
00498 Random rand(CkMyPe());
00499 rand.reorder(gridPeOrder,numGridPes);
00500 rand.reorder(transPeOrder,numTransPes);
00501 }
00502
00503 int sum_npes = numTransPes + numGridPes;
00504 int max_npes = (numTransPes > numGridPes)?numTransPes:numGridPes;
00505
00506 #if USE_TOPOMAP
00507 PatchMap * pmap = PatchMap::Object();
00508
00509 int patch_pes = pmap->numNodesWithPatches();
00510 TopoManager tmgr;
00511 if(tmgr.hasMultipleProcsPerNode())
00512 patch_pes *= 2;
00513
00514 bool done = false;
00515 #ifndef USE_COMM_LIB
00516 if(CkNumPes() > 2*sum_npes + patch_pes) {
00517 done = generateBGLORBPmePeList(transPeMap, numTransPes);
00518 done &= generateBGLORBPmePeList(gridPeMap, numGridPes, transPeMap, numTransPes);
00519 }
00520 else
00521 #endif
00522 if(CkNumPes() > 2 *max_npes + patch_pes) {
00523 done = generateBGLORBPmePeList(transPeMap, max_npes);
00524 gridPeMap = transPeMap;
00525 }
00526
00527 if (!done)
00528 #endif
00529 {
00530 //generatePmePeList(transPeMap, max_npes);
00531 //gridPeMap = transPeMap;
00532 generatePmePeList2(gridPeMap, numGridPes, transPeMap, numTransPes);
00533 }
00534
00535 #ifdef USE_COMM_LIB
00536 if(CkMyPe() == 0) {
00537 ComlibInstanceHandle cinst1 = CkCreateComlibInstance();
00538 EachToManyMulticastStrategy *strat = new
00539 EachToManyMulticastStrategy(USE_DIRECT, numGridPes,
00540 gridPeMap, numTransPes, transPeMap);
00541 cinst1.setStrategy(strat);
00542
00543 ComlibInstanceHandle cinst2 = CkCreateComlibInstance();
00544 strat = new EachToManyMulticastStrategy(USE_DIRECT, numTransPes, transPeMap
00545 , numGridPes, gridPeMap);
00546 cinst2.setStrategy(strat);
00547 ComlibDoneCreating();
00548 }
00549 #endif
00550
00551 myGridPe = -1;
00552 int i = 0;
00553 for ( i=0; i<CkNumPes(); ++i )
00554 isPmeFlag[i] = 0;
00555 for ( i=0; i<numGridPes; ++i ) {
00556 if ( gridPeMap[i] == CkMyPe() ) myGridPe = i;
00557 isPmeFlag[gridPeMap[i]] |= 1;
00558 }
00559 myTransPe = -1;
00560 for ( i=0; i<numTransPes; ++i ) {
00561 if ( transPeMap[i] == CkMyPe() ) myTransPe = i;
00562 isPmeFlag[transPeMap[i]] |= 2;
00563 }
00564
00565 if ( ! CkMyPe() ) {
00566 iout << iINFO << "PME GRID LOCATIONS:";
00567 int i;
00568 for ( i=0; i<numGridPes && i<10; ++i ) {
00569 iout << " " << gridPeMap[i];
00570 }
00571 if ( i < numGridPes ) iout << " ...";
00572 iout << "\n" << endi;
00573 iout << iINFO << "PME TRANS LOCATIONS:";
00574 for ( i=0; i<numTransPes && i<10; ++i ) {
00575 iout << " " << transPeMap[i];
00576 }
00577 if ( i < numTransPes ) iout << " ...";
00578 iout << "\n" << endi;
00579 }
00580
00581 } // ! usePencils
00582
00583 myGrid.K1 = simParams->PMEGridSizeX;
00584 myGrid.K2 = simParams->PMEGridSizeY;
00585 myGrid.K3 = simParams->PMEGridSizeZ;
00586 myGrid.order = simParams->PMEInterpOrder;
00587 myGrid.dim2 = myGrid.K2;
00588 myGrid.dim3 = 2 * (myGrid.K3/2 + 1);
00589
00590 if ( ! usePencils ) {
00591 myGrid.block1 = ( myGrid.K1 + numGridPes - 1 ) / numGridPes;
00592 myGrid.block2 = ( myGrid.K2 + numTransPes - 1 ) / numTransPes;
00593 myGrid.block3 = myGrid.dim3 / 2; // complex
00594 }
00595
00596 if ( usePencils ) {
00597 myGrid.block1 = ( myGrid.K1 + xBlocks - 1 ) / xBlocks;
00598 myGrid.block2 = ( myGrid.K2 + yBlocks - 1 ) / yBlocks;
00599 myGrid.block3 = ( myGrid.K3/2 + 1 + zBlocks - 1 ) / zBlocks; // complex
00600
00601 if ( CkMyPe() == 0 ) {
00602 int basepe = 0; int npe = CkNumPes();
00603 if ( npe > xBlocks*yBlocks &&
00604 npe > xBlocks*zBlocks &&
00605 npe > yBlocks*zBlocks ) {
00606 // avoid node 0
00607 ++basepe;
00608 --npe;
00609 }
00610
00611 zPencil = CProxy_PmeZPencil::ckNew(); // (xBlocks,yBlocks,1);
00612 yPencil = CProxy_PmeYPencil::ckNew(); // (xBlocks,1,zBlocks);
00613 xPencil = CProxy_PmeXPencil::ckNew(); // (1,yBlocks,zBlocks);
00614
00615 // decide which pes to use by bit reversal and patch use
00616 int i;
00617 int ncpus = CkNumPes();
00618
00619 // find next highest power of two
00620 int npow2 = 1; int nbits = 0;
00621 while ( npow2 < ncpus ) { npow2 *= 2; nbits += 1; }
00622
00623 // build bit reversal sequence
00624 SortableResizeArray<int> patches, nopatches, pmeprocs;
00625 PatchMap *pmap = PatchMap::Object();
00626 i = 0;
00627 for ( int icpu=0; icpu<ncpus; ++icpu ) {
00628 int ri;
00629 for ( ri = ncpus; ri >= ncpus; ++i ) {
00630 ri = 0;
00631 int pow2 = 1;
00632 int rpow2 = npow2 / 2;
00633 for ( int j=0; j<nbits; ++j ) {
00634 ri += rpow2 * ( ( i / pow2 ) % 2 );
00635 pow2 *= 2; rpow2 /= 2;
00636 }
00637 }
00638 // seq[icpu] = ri;
00639 if ( ri ) { // keep 0 for special case
00640 if ( pmap->numPatchesOnNode(ri) ) patches.add(ri);
00641 else nopatches.add(ri);
00642 }
00643 }
00644
00645 // only use zero if it eliminates overloading or has patches
00646 int useZero = 0;
00647 int npens = xBlocks*yBlocks;
00648 if ( npens % ncpus == 0 ) useZero = 1;
00649 if ( npens == nopatches.size() + 1 ) useZero = 1;
00650 npens += xBlocks*zBlocks;
00651 if ( npens % ncpus == 0 ) useZero = 1;
00652 if ( npens == nopatches.size() + 1 ) useZero = 1;
00653 npens += yBlocks*zBlocks;
00654 if ( npens % ncpus == 0 ) useZero = 1;
00655 if ( npens == nopatches.size() + 1 ) useZero = 1;
00656
00657 // add nopatches then patches in reversed order
00658 for ( i=nopatches.size()-1; i>=0; --i ) pmeprocs.add(nopatches[i]);
00659 if ( useZero && ! pmap->numPatchesOnNode(0) ) pmeprocs.add(0);
00660 for ( i=patches.size()-1; i>=0; --i ) pmeprocs.add(patches[i]);
00661 if ( pmap->numPatchesOnNode(0) ) pmeprocs.add(0);
00662
00663 int pe = 0;
00664 int npes = pmeprocs.size();
00665 SortableResizeArray<int> zprocs(xBlocks*yBlocks);
00666 for ( i=0; i<xBlocks*yBlocks; ++i, ++pe ) zprocs[i] = pmeprocs[pe%npes];
00667 zprocs.sort();
00668 SortableResizeArray<int> yprocs(xBlocks*zBlocks);
00669 for ( i=0; i<xBlocks*zBlocks; ++i, ++pe ) yprocs[i] = pmeprocs[pe%npes];
00670 yprocs.sort();
00671 SortableResizeArray<int> xprocs(yBlocks*zBlocks);
00672 for ( i=0; i<yBlocks*zBlocks; ++i, ++pe ) xprocs[i] = pmeprocs[pe%npes];
00673 xprocs.sort();
00674
00675 pencilPMEProcessors = new char [CkNumPes()];
00676 memset (pencilPMEProcessors, 0, sizeof(char) * CkNumPes());
00677
00678 int x,y,z;
00679
00680 iout << iINFO << "PME Z PENCIL LOCATIONS:";
00681 for ( i=0; i<zprocs.size() && i<10; ++i ) {
00682 iout << " " << zprocs[i];
00683 }
00684 if ( i < zprocs.size() ) iout << " ...";
00685 iout << "\n" << endi;
00686
00687 for (pe=0, x = 0; x < xBlocks; ++x)
00688 for (y = 0; y < yBlocks; ++y, ++pe ) {
00689 zPencil(x,y,0).insert(zprocs[pe]);
00690 pencilPMEProcessors[zprocs[pe]] = 1;
00691 }
00692 zPencil.doneInserting();
00693
00694 iout << iINFO << "PME Y PENCIL LOCATIONS:";
00695 for ( i=0; i<yprocs.size() && i<10; ++i ) {
00696 iout << " " << yprocs[i];
00697 }
00698 if ( i < yprocs.size() ) iout << " ...";
00699 iout << "\n" << endi;
00700
00701 for (pe=0, z = 0; z < zBlocks; ++z )
00702 for (x = 0; x < xBlocks; ++x, ++pe ) {
00703 yPencil(x,0,z).insert(yprocs[pe]);
00704 pencilPMEProcessors[yprocs[pe]] = 1;
00705 }
00706 yPencil.doneInserting();
00707
00708 iout << iINFO << "PME X PENCIL LOCATIONS:";
00709 for ( i=0; i<xprocs.size() && i<10; ++i ) {
00710 iout << " " << xprocs[i];
00711 }
00712 if ( i < xprocs.size() ) iout << " ...";
00713 iout << "\n" << endi;
00714
00715 for (pe=0, y = 0; y < yBlocks; ++y )
00716 for (z = 0; z < zBlocks; ++z, ++pe ) {
00717 xPencil(0,y,z).insert(xprocs[pe]);
00718 pencilPMEProcessors[xprocs[pe]] = 1;
00719 }
00720 xPencil.doneInserting();
00721
00722 pmeProxy.recvArrays(xPencil,yPencil,zPencil);
00723 PmePencilInitMsgData msgdata;
00724 msgdata.grid = myGrid;
00725 msgdata.xBlocks = xBlocks;
00726 msgdata.yBlocks = yBlocks;
00727 msgdata.zBlocks = zBlocks;
00728 msgdata.xPencil = xPencil;
00729 msgdata.yPencil = yPencil;
00730 msgdata.zPencil = zPencil;
00731 msgdata.pmeProxy = pmeProxyDir;
00732 xPencil.init(new PmePencilInitMsg(msgdata));
00733 yPencil.init(new PmePencilInitMsg(msgdata));
00734 zPencil.init(new PmePencilInitMsg(msgdata));
00735 }
00736 return; // continue in initialize_pencils() at next startup stage
00737 }
00738
00739
00740 int pe;
00741 int nx = 0;
00742 for ( pe = 0; pe < numGridPes; ++pe ) {
00743 localInfo[pe].x_start = nx;
00744 nx += myGrid.block1;
00745 if ( nx > myGrid.K1 ) nx = myGrid.K1;
00746 localInfo[pe].nx = nx - localInfo[pe].x_start;
00747 }
00748 int ny = 0;
00749 for ( pe = 0; pe < numTransPes; ++pe ) {
00750 localInfo[pe].y_start_after_transpose = ny;
00751 ny += myGrid.block2;
00752 if ( ny > myGrid.K2 ) ny = myGrid.K2;
00753 localInfo[pe].ny_after_transpose =
00754 ny - localInfo[pe].y_start_after_transpose;
00755 }
00756
00757 { // decide how many pes this node exchanges charges with
00758
00759 PatchMap *patchMap = PatchMap::Object();
00760 Lattice lattice = simParams->lattice;
00761 BigReal sysdima = lattice.a_r().unit() * lattice.a();
00762 BigReal cutoff = simParams->cutoff;
00763 BigReal patchdim = simParams->patchDimension;
00764 int numPatches = patchMap->numPatches();
00765 int numNodes = CkNumPes();
00766 int *source_flags = new int[numNodes];
00767 int node;
00768 for ( node=0; node<numNodes; ++node ) {
00769 source_flags[node] = 0;
00770 recipPeDest[node] = 0;
00771 }
00772
00773 // // make sure that we don't get ahead of ourselves on this node
00774 // if ( CkMyPe() < numPatches && myRecipPe >= 0 ) {
00775 // source_flags[CkMyPe()] = 1;
00776 // recipPeDest[myRecipPe] = 1;
00777 // }
00778
00779 for ( int pid=0; pid < numPatches; ++pid ) {
00780 int pnode = patchMap->node(pid);
00781 BigReal minx = patchMap->min_a(pid);
00782 BigReal maxx = patchMap->max_a(pid);
00783 BigReal margina = 0.5 * ( patchdim - cutoff ) / sysdima;
00784 // min1 (max1) is smallest (largest) grid line for this patch
00785 int min1 = ((int) floor(myGrid.K1 * (minx - margina))) - myGrid.order + 1;
00786 int max1 = ((int) floor(myGrid.K1 * (maxx + margina)));
00787 for ( int i=min1; i<=max1; ++i ) {
00788 int ix = i;
00789 while ( ix >= myGrid.K1 ) ix -= myGrid.K1;
00790 while ( ix < 0 ) ix += myGrid.K1;
00791 // set source_flags[pnode] if this patch sends to our node
00792 if ( myGridPe >= 0 && ix >= localInfo[myGridPe].x_start &&
00793 ix < localInfo[myGridPe].x_start + localInfo[myGridPe].nx ) {
00794 source_flags[pnode] = 1;
00795 }
00796 // set dest_flags[] for node that our patch sends to
00797 if ( pnode == CkMyPe() ) {
00798 recipPeDest[ix / myGrid.block1] = 1;
00799 }
00800 }
00801 }
00802
00803 numSources = 0;
00804 numDestRecipPes = 0;
00805 for ( node=0; node<numNodes; ++node ) {
00806 if ( source_flags[node] ) ++numSources;
00807 if ( recipPeDest[node] ) ++numDestRecipPes;
00808 }
00809
00810 #if 0
00811 if ( numSources ) {
00812 iout << iINFO << "PME " << CkMyPe() << " sources:";
00813 for ( node=0; node<numNodes; ++node ) {
00814 if ( source_flags[node] ) iout << " " << node;
00815 }
00816 iout << "\n" << endi;
00817 }
00818 #endif
00819
00820 delete [] source_flags;
00821
00822 // CkPrintf("PME on node %d has %d sources and %d destinations\n",
00823 // CkMyPe(), numSources, numDestRecipPes);
00824
00825 } // decide how many pes this node exchanges charges with (end)
00826
00827 ungrid_count = numDestRecipPes;
00828
00829 sendTransBarrier_received = 0;
00830
00831 if ( myGridPe < 0 && myTransPe < 0 ) return;
00832 // the following only for nodes doing reciprocal sum
00833
00834 if ( myTransPe >= 0 ) {
00835 int k2_start = localInfo[myTransPe].y_start_after_transpose;
00836 int k2_end = k2_start + localInfo[myTransPe].ny_after_transpose;
00837 myKSpace = new PmeKSpace(myGrid, k2_start, k2_end, 0, myGrid.dim3/2);
00838 }
00839
00840 int local_size = myGrid.block1 * myGrid.K2 * myGrid.dim3;
00841 int local_size_2 = myGrid.block2 * myGrid.K1 * myGrid.dim3;
00842 if ( local_size < local_size_2 ) local_size = local_size_2;
00843 qgrid = new float[local_size*numGrids];
00844 if ( numGridPes > 1 || numTransPes > 1 ) {
00845 kgrid = new float[local_size*numGrids];
00846 } else {
00847 kgrid = qgrid;
00848 }
00849 qgrid_size = local_size;
00850
00851 if ( myGridPe >= 0 ) {
00852 qgrid_start = localInfo[myGridPe].x_start * myGrid.K2 * myGrid.dim3;
00853 qgrid_len = localInfo[myGridPe].nx * myGrid.K2 * myGrid.dim3;
00854 fgrid_start = localInfo[myGridPe].x_start * myGrid.K2;
00855 fgrid_len = localInfo[myGridPe].nx * myGrid.K2;
00856 }
00857
00858 int n[3]; n[0] = myGrid.K1; n[1] = myGrid.K2; n[2] = myGrid.K3;
00859
00860 #ifdef NAMD_FFTW
00861 CmiLock(fftw_plan_lock);
00862
00863 work = new fftw_complex[n[0]];
00864
00865 if ( ! CkMyPe() ) iout << iINFO << "Optimizing 4 FFT steps. 1..." << endi;
00866 if ( myGridPe >= 0 ) {
00867 forward_plan_yz = rfftwnd_create_plan_specific(2, n+1, FFTW_REAL_TO_COMPLEX,
00868 ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE )
00869 | FFTW_IN_PLACE | FFTW_USE_WISDOM, qgrid, 1, 0, 0);
00870 }
00871 if ( ! CkMyPe() ) iout << " 2..." << endi;
00872 if ( myTransPe >= 0 ) {
00873 forward_plan_x = fftw_create_plan_specific(n[0], FFTW_REAL_TO_COMPLEX,
00874 ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE )
00875 | FFTW_IN_PLACE | FFTW_USE_WISDOM, (fftw_complex *) kgrid,
00876 localInfo[myTransPe].ny_after_transpose * myGrid.dim3 / 2, work, 1);
00877 }
00878 if ( ! CkMyPe() ) iout << " 3..." << endi;
00879 if ( myTransPe >= 0 ) {
00880 backward_plan_x = fftw_create_plan_specific(n[0], FFTW_COMPLEX_TO_REAL,
00881 ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE )
00882 | FFTW_IN_PLACE | FFTW_USE_WISDOM, (fftw_complex *) kgrid,
00883 localInfo[myTransPe].ny_after_transpose * myGrid.dim3 / 2, work, 1);
00884 }
00885 if ( ! CkMyPe() ) iout << " 4..." << endi;
00886 if ( myGridPe >= 0 ) {
00887 backward_plan_yz = rfftwnd_create_plan_specific(2, n+1, FFTW_COMPLEX_TO_REAL,
00888 ( simParams->FFTWEstimate ? FFTW_ESTIMATE : FFTW_MEASURE )
00889 | FFTW_IN_PLACE | FFTW_USE_WISDOM, qgrid, 1, 0, 0);
00890 }
00891 if ( ! CkMyPe() ) iout << " Done.\n" << endi;
00892
00893 CmiUnlock(fftw_plan_lock);
00894 #else
00895 NAMD_die("Sorry, FFTW must be compiled in to use PME.");
00896 #endif
00897
00898 if ( myGridPe >= 0 && numSources == 0 )
00899 NAMD_bug("PME grid elements exist without sources.");
00900 grid_count = numSources;
00901 memset( (void*) qgrid, 0, qgrid_size * numGrids * sizeof(float) );
00902 trans_count = numGridPes;
00903 }
|
|
|
Definition at line 906 of file ComputePme.C. References Lattice::a(), Lattice::a_r(), Lattice::b(), Lattice::b_r(), BigReal, PmeGrid::block1, PmeGrid::block2, SimParameters::cutoff, j, PmeGrid::K1, PmeGrid::K2, SimParameters::lattice, PatchMap::max_a(), PatchMap::max_b(), PatchMap::min_a(), PatchMap::min_b(), PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), Node::Object(), PmeGrid::order, SimParameters::patchDimension, Node::simParameters, simParams, and Vector::unit(). 00906 {
00907 delete msg;
00908 if ( ! usePencils ) return;
00909
00910 SimParameters *simParams = Node::Object()->simParameters;
00911
00912 PatchMap *patchMap = PatchMap::Object();
00913 Lattice lattice = simParams->lattice;
00914 BigReal sysdima = lattice.a_r().unit() * lattice.a();
00915 BigReal sysdimb = lattice.b_r().unit() * lattice.b();
00916 BigReal cutoff = simParams->cutoff;
00917 BigReal patchdim = simParams->patchDimension;
00918 int numPatches = patchMap->numPatches();
00919
00920 pencilActive = new char[xBlocks*yBlocks];
00921 for ( int i=0; i<xBlocks; ++i ) {
00922 for ( int j=0; j<yBlocks; ++j ) {
00923 pencilActive[i*yBlocks+j] = 0;
00924 }
00925 }
00926
00927 for ( int pid=0; pid < numPatches; ++pid ) {
00928 int pnode = patchMap->node(pid);
00929 if ( pnode != CkMyPe() ) continue;
00930
00931 BigReal minx = patchMap->min_a(pid);
00932 BigReal maxx = patchMap->max_a(pid);
00933 BigReal margina = 0.5 * ( patchdim - cutoff ) / sysdima;
00934 // min1 (max1) is smallest (largest) grid line for this patch
00935 int min1 = ((int) floor(myGrid.K1 * (minx - margina))) - myGrid.order + 1;
00936 int max1 = ((int) floor(myGrid.K1 * (maxx + margina)));
00937
00938 BigReal miny = patchMap->min_b(pid);
00939 BigReal maxy = patchMap->max_b(pid);
00940 BigReal marginb = 0.5 * ( patchdim - cutoff ) / sysdimb;
00941 // min2 (max2) is smallest (largest) grid line for this patch
00942 int min2 = ((int) floor(myGrid.K2 * (miny - marginb))) - myGrid.order + 1;
00943 int max2 = ((int) floor(myGrid.K2 * (maxy + marginb)));
00944
00945 for ( int i=min1; i<=max1; ++i ) {
00946 int ix = i;
00947 while ( ix >= myGrid.K1 ) ix -= myGrid.K1;
00948 while ( ix < 0 ) ix += myGrid.K1;
00949 for ( int j=min2; j<=max2; ++j ) {
00950 int jy = j;
00951 while ( jy >= myGrid.K2 ) jy -= myGrid.K2;
00952 while ( jy < 0 ) jy += myGrid.K2;
00953 pencilActive[(ix / myGrid.block1)*yBlocks + (jy / myGrid.block2)] = 1;
00954 }
00955 }
00956 }
00957
00958 numPencilsActive = 0;
00959 for ( int i=0; i<xBlocks; ++i ) {
00960 for ( int j=0; j<yBlocks; ++j ) {
00961 if ( pencilActive[i*yBlocks+j] ) {
00962 ++numPencilsActive;
00963 zPencil(i,j,0).dummyRecvGrid(CkMyPe(),0);
00964 }
00965 }
00966 }
00967 //if ( numPencilsActive ) {
00968 // CkPrintf("node %d sending to %d pencils\n", CkMyPe(), numPencilsActive);
00969 //}
00970
00971 ungrid_count = numPencilsActive;
00972 }
|
|
|
Definition at line 329 of file ComputePme.C. References pencilPMEProcessors. 00329 {
00330 return ( usePencils ? pencilPMEProcessors[p] : isPmeFlag[p] );
00331 }
|
|
||||||||||||||||
|
Definition at line 362 of file ComputePme.C. 00363 {
00364 xPencil = x; yPencil = y; zPencil = z;
00365 }
|
|
|
Definition at line 1007 of file ComputePme.C. References PmeGrid::dim3, PmeGridMsg::fgrid, PmeGridMsg::lattice, NAMD_bug(), PmeGridMsg::qgrid, PmeGridMsg::sequence, PmeGridMsg::zlist, and PmeGridMsg::zlistlen. Referenced by ComputePme::sendPencils(). 01007 {
01008 // CkPrintf("recvGrid from %d on Pe(%d)\n",msg->sourceNode,CkMyPe());
01009 if ( grid_count == 0 ) {
01010 NAMD_bug("Message order failure in ComputePmeMgr::recvGrid\n");
01011 }
01012 if ( grid_count == numSources ) {
01013 lattice = msg->lattice;
01014 sequence = msg->sequence;
01015 }
01016
01017 int zdim = myGrid.dim3;
01018 int zlistlen = msg->zlistlen;
01019 int *zlist = msg->zlist;
01020 float *qmsg = msg->qgrid;
01021 for ( int g=0; g<numGrids; ++g ) {
01022 char *f = msg->fgrid + fgrid_len * g;
01023 float *q = qgrid + qgrid_size * g;
01024 for ( int i=0; i<fgrid_len; ++i ) {
01025 if ( f[i] ) {
01026 for ( int k=0; k<zlistlen; ++k ) {
01027 q[zlist[k]] += *(qmsg++);
01028 }
01029 }
01030 q += zdim;
01031 }
01032 }
01033
01034 gridmsg_reuse[numSources-grid_count] = msg;
01035 --grid_count;
01036
01037 if ( grid_count == 0 ) {
01038 pmeProxyDir[CkMyPe()].gridCalc1();
01039 if ( useBarrier ) pmeProxyDir[0].sendTransBarrier();
01040 }
01041 }
|
|
|
Definition at line 1115 of file ComputePme.C. References PmeGrid::dim3, PmeTransMsg::lattice, PmeTransMsg::nx, LocalPmeInfo::ny_after_transpose, PmeTransMsg::qgrid, PmeTransMsg::sequence, and PmeTransMsg::x_start. 01115 {
01116 // CkPrintf("recvTrans on Pe(%d)\n",CkMyPe());
01117 if ( trans_count == numGridPes ) {
01118 lattice = msg->lattice;
01119 sequence = msg->sequence;
01120 }
01121
01122 int zdim = myGrid.dim3;
01123 // int y_start = localInfo[myTransPe].y_start_after_transpose;
01124 int ny = localInfo[myTransPe].ny_after_transpose;
01125 int x_start = msg->x_start;
01126 int nx = msg->nx;
01127 for ( int g=0; g<numGrids; ++g ) {
01128 CmiMemcpy((void*)(kgrid + qgrid_size * g + x_start*ny*zdim),
01129 (void*)(msg->qgrid + nx*ny*zdim*g), nx*ny*zdim*sizeof(float));
01130 }
01131
01132 delete msg;
01133 --trans_count;
01134
01135 if ( trans_count == 0 ) {
01136 pmeProxyDir[CkMyPe()].gridCalc2();
01137 }
01138 }
|
|
|
Definition at line 1320 of file ComputePme.C. References ComputePme::copyPencils(), ComputePme::copyResults(), and NAMD_bug(). 01320 {
01321 // CkPrintf("recvUngrid on Pe(%d)\n",CkMyPe());
01322 if ( ungrid_count == 0 ) {
01323 NAMD_bug("Message order failure in ComputePmeMgr::recvUngrid\n");
01324 }
01325
01326 if ( usePencils ) pmeCompute->copyPencils(msg);
01327 else pmeCompute->copyResults(msg);
01328 delete msg;
01329 --ungrid_count;
01330
01331 if ( ungrid_count == 0 ) {
01332 pmeProxyDir[CkMyPe()].ungridCalc();
01333 }
01334 }
|
|
|
Definition at line 1221 of file ComputePme.C. References PmeGrid::dim3, PmeUntransMsg::evir, PmeGrid::K2, LocalPmeInfo::nx, PmeUntransMsg::ny, PmeUntransMsg::qgrid, and PmeUntransMsg::y_start. 01221 {
01222 // CkPrintf("recvUntrans on Pe(%d)\n",CkMyPe());
01223 if ( untrans_count == numTransPes ) {
01224 for ( int g=0; g<numGrids; ++g ) {
01225 recip_evir[g] = 0.;
01226 }
01227 }
01228
01229 #if CMK_BLUEGENEL
01230 CmiNetworkProgressAfter (0);
01231 #endif
01232
01233 int g;
01234 for ( g=0; g<numGrids; ++g ) {
01235 recip_evir[g] += msg->evir[g];
01236 }
01237
01238 int zdim = myGrid.dim3;
01239 // int x_start = localInfo[myGridPe].x_start;
01240 int nx = localInfo[myGridPe].nx;
01241 int y_start = msg->y_start;
01242 int ny = msg->ny;
01243 int slicelen = myGrid.K2 * zdim;
01244 int cpylen = ny * zdim;
01245 for ( g=0; g<numGrids; ++g ) {
01246 float *q = qgrid + qgrid_size * g + y_start * zdim;
01247 float *qmsg = msg->qgrid + nx * cpylen * g;
01248 for ( int x = 0; x < nx; ++x ) {
01249 CmiMemcpy((void*)q, (void*)qmsg, cpylen*sizeof(float));
01250 q += slicelen;
01251 qmsg += cpylen;
01252 }
01253 }
01254
01255 delete msg;
01256 --untrans_count;
01257
01258 if ( untrans_count == 0 ) {
01259 pmeProxyDir[CkMyPe()].gridCalc3();
01260 }
01261 }
|
|
|
Definition at line 1003 of file ComputePme.C. References ComputePme::sendData(). 01003 {
01004 pmeCompute->sendData(numGridPes,gridPeOrder,recipPeDest,gridPeMap);
01005 }
|
|
|
Definition at line 1066 of file ComputePme.C. References PmeGrid::dim3, j, PmeGrid::K2, PmeTransMsg::lattice, PmeTransMsg::nx, LocalPmeInfo::nx, LocalPmeInfo::ny_after_transpose, PME_TRANS_PRIORITY, PmeTransMsg::qgrid, PmeTransMsg::sequence, SET_PRIORITY, PmeTransMsg::sourceNode, PmeTransMsg::x_start, LocalPmeInfo::x_start, and LocalPmeInfo::y_start_after_transpose. 01066 {
01067 // CkPrintf("sendTrans on %d\n",myTransPe);
01068
01069 // send data for transpose
01070 int zdim = myGrid.dim3;
01071 int nx = localInfo[myGridPe].nx;
01072 int x_start = localInfo[myGridPe].x_start;
01073 int slicelen = myGrid.K2 * zdim;
01074
01075 #ifdef USE_COMM_LIB
01076 ComlibInstanceHandle cinst1 = CkGetComlibInstance(0);
01077 cinst1.beginIteration();
01078 #endif
01079
01080 #if CMK_BLUEGENEL
01081 CmiNetworkProgressAfter (0);
01082 #endif
01083
01084 for (int j=0; j<numTransPes; j++) {
01085 int pe = transPeOrder[j]; // different order on each node
01086 LocalPmeInfo &li = localInfo[pe];
01087 int cpylen = li.ny_after_transpose * zdim;
01088 PmeTransMsg *newmsg = new (nx * cpylen * numGrids,
01089 PRIORITY_SIZE) PmeTransMsg;
01090 newmsg->sourceNode = myGridPe;
01091 newmsg->lattice = lattice;
01092 newmsg->x_start = x_start;
01093 newmsg->nx = nx;
01094 for ( int g=0; g<numGrids; ++g ) {
01095 float *q = qgrid + qgrid_size * g + li.y_start_after_transpose * zdim;
01096 float *qmsg = newmsg->qgrid + nx * cpylen * g;
01097 for ( int x = 0; x < nx; ++x ) {
01098 CmiMemcpy((void*)qmsg, (void*)q, cpylen*sizeof(float));
01099 q += slicelen;
01100 qmsg += cpylen;
01101 }
01102 }
01103 newmsg->sequence = sequence;
01104 SET_PRIORITY(newmsg,sequence,PME_TRANS_PRIORITY)
01105 pmeProxy[transPeMap[pe]].recvTrans(newmsg);
01106 }
01107
01108 untrans_count = numTransPes;
01109
01110 #ifdef USE_COMM_LIB
01111 cinst1.endIteration();
01112 #endif
01113 }
|
|
|
Definition at line 1056 of file ComputePme.C. 01056 {
01057 sendTransBarrier_received += 1;
01058 // CkPrintf("sendTransBarrier on %d %d\n",myGridPe,numGridPes-sendTransBarrier_received);
01059 if ( sendTransBarrier_received < numGridPes ) return;
01060 sendTransBarrier_received = 0;
01061 for ( int i=0; i<numGridPes; ++i ) {
01062 pmeProxyDir[gridPeMap[i]].sendTrans();
01063 }
01064 }
|
|
|
Definition at line 1278 of file ComputePme.C. References PmeGrid::dim3, PmeGridMsg::evir, PmeGridMsg::fgrid, j, PmeGridMsg::len, PME_UNGRID_PRIORITY, PmeGridMsg::qgrid, SET_PRIORITY, PmeGridMsg::sourceNode, PmeGridMsg::start, PmeGridMsg::zlist, and PmeGridMsg::zlistlen. 01278 {
01279
01280 for ( int j=0; j<numSources; ++j ) {
01281 // int msglen = qgrid_len;
01282 PmeGridMsg *newmsg = gridmsg_reuse[j];
01283 int pe = newmsg->sourceNode;
01284 if ( j == 0 ) { // only need these once
01285 for ( int g=0; g<numGrids; ++g ) {
01286 newmsg->evir[g] = recip_evir[g];
01287 }
01288 } else {
01289 for ( int g=0; g<numGrids; ++g ) {
01290 newmsg->evir[g] = 0.;
01291 }
01292 }
01293 int zdim = myGrid.dim3;
01294 int flen = newmsg->len;
01295 int fstart = newmsg->start;
01296 int zlistlen = newmsg->zlistlen;
01297 int *zlist = newmsg->zlist;
01298 float *qmsg = newmsg->qgrid;
01299 for ( int g=0; g<numGrids; ++g ) {
01300 char *f = newmsg->fgrid + fgrid_len * g;
01301 float *q = qgrid + qgrid_size * g + (fstart-fgrid_start) * zdim;
01302 for ( int i=0; i<flen; ++i ) {
01303 if ( f[i] ) {
01304 for ( int k=0; k<zlistlen; ++k ) {
01305 *(qmsg++) = q[zlist[k]];
01306 }
01307 }
01308 q += zdim;
01309 }
01310 }
01311 newmsg->sourceNode = myGridPe;
01312
01313 SET_PRIORITY(newmsg,sequence,PME_UNGRID_PRIORITY)
01314 pmeProxyDir[pe].recvUngrid(newmsg);
01315 }
01316 grid_count = numSources;
01317 memset( (void*) qgrid, 0, qgrid_size * numGrids * sizeof(float) );
01318 }
|
|
|
Definition at line 1174 of file ComputePme.C. References PmeGrid::dim3, PmeUntransMsg::evir, j, LocalPmeInfo::nx, PmeUntransMsg::ny, LocalPmeInfo::ny_after_transpose, PME_UNTRANS_PRIORITY, PmeUntransMsg::qgrid, SET_PRIORITY, PmeUntransMsg::sourceNode, LocalPmeInfo::x_start, PmeUntransMsg::y_start, and LocalPmeInfo::y_start_after_transpose. 01174 {
01175
01176 int zdim = myGrid.dim3;
01177 int y_start = localInfo[myTransPe].y_start_after_transpose;
01178 int ny = localInfo[myTransPe].ny_after_transpose;
01179
01180 #ifdef USE_COMM_LIB
01181 ComlibInstanceHandle cinst2 = CkGetComlibInstance(1);
01182 cinst2.beginIteration();
01183 #endif
01184
01185 #if CMK_BLUEGENEL
01186 CmiNetworkProgressAfter (0);
01187 #endif
01188
01189 // send data for reverse transpose
01190 for (int j=0; j<numGridPes; j++) {
01191 int pe = gridPeOrder[j]; // different order on each node
01192 LocalPmeInfo &li = localInfo[pe];
01193 int x_start =li.x_start;
01194 int nx = li.nx;
01195 PmeUntransMsg *newmsg = new (nx*ny*zdim*numGrids,numGrids,
01196 PRIORITY_SIZE) PmeUntransMsg;
01197 newmsg->sourceNode = myTransPe;
01198 newmsg->y_start = y_start;
01199 newmsg->ny = ny;
01200 for ( int g=0; g<numGrids; ++g ) {
01201 if ( j == 0 ) { // only need these once
01202 newmsg->evir[g] = recip_evir2[g];
01203 } else {
01204 newmsg->evir[g] = 0.;
01205 }
01206 CmiMemcpy((void*)(newmsg->qgrid+nx*ny*zdim*g),
01207 (void*)(kgrid + qgrid_size*g + x_start*ny*zdim),
01208 nx*ny*zdim*sizeof(float));
01209 }
01210 SET_PRIORITY(newmsg,sequence,PME_UNTRANS_PRIORITY)
01211 pmeProxy[gridPeMap[pe]].recvUntrans(newmsg);
01212 }
01213
01214 #ifdef USE_COMM_LIB
01215 cinst2.endIteration();
01216 #endif
01217
01218 trans_count = numGridPes;
01219 }
|
|
|
Definition at line 244 of file ComputePme.C. References ComputePme::setMgr(). 00244 { pmeCompute = c; c->setMgr(this); }
|
|
|
Definition at line 1336 of file ComputePme.C. References ComputePme::ungridForces(). Referenced by ComputePme::doWork(). 01336 {
01337 // CkPrintf("ungridCalc on Pe(%d)\n",CkMyPe());
01338
01339 pmeCompute->ungridForces();
01340
01341 ungrid_count = (usePencils ? numPencilsActive : numDestRecipPes );
01342 }
|
|
|
Definition at line 221 of file ComputePme.C. |
|
|
Definition at line 319 of file ComputePme.C. Referenced by ComputePmeMgr(), initialize(), and ~ComputePmeMgr(). |
1.3.9.1