#include <WorkDistrib.h>

Inheritance diagram for WorkDistrib:

Classes
struct	pe_sortop_compact

struct	pe_sortop_diffuse

Public Member Functions
	WorkDistrib ()

	~WorkDistrib (void)

void	enqueueWork (LocalWorkMsg *msg)

void	enqueueExcls (LocalWorkMsg *msg)

void	enqueueBonds (LocalWorkMsg *msg)

void	enqueueAngles (LocalWorkMsg *msg)

void	enqueueDihedrals (LocalWorkMsg *msg)

void	enqueueImpropers (LocalWorkMsg *msg)

void	enqueueThole (LocalWorkMsg *msg)

void	enqueueAniso (LocalWorkMsg *msg)

void	enqueueCrossterms (LocalWorkMsg *msg)

void	enqueueOneFourNbThole (LocalWorkMsg *msg)

void	enqueueGromacsPair (LocalWorkMsg *msg)

void	enqueuePme (LocalWorkMsg *msg)

void	enqueueSelfA1 (LocalWorkMsg *msg)

void	enqueueSelfA2 (LocalWorkMsg *msg)

void	enqueueSelfA3 (LocalWorkMsg *msg)

void	enqueueSelfB1 (LocalWorkMsg *msg)

void	enqueueSelfB2 (LocalWorkMsg *msg)

void	enqueueSelfB3 (LocalWorkMsg *msg)

void	enqueueWorkA1 (LocalWorkMsg *msg)

void	enqueueWorkA2 (LocalWorkMsg *msg)

void	enqueueWorkA3 (LocalWorkMsg *msg)

void	enqueueWorkB1 (LocalWorkMsg *msg)

void	enqueueWorkB2 (LocalWorkMsg *msg)

void	enqueueWorkB3 (LocalWorkMsg *msg)

void	enqueueWorkC (LocalWorkMsg *msg)

void	enqueueCUDA (LocalWorkMsg *msg)

void	enqueueCUDAP2 (LocalWorkMsg *msg)

void	enqueueCUDAP3 (LocalWorkMsg *msg)

void	finishCUDAPatch (FinishWorkMsg *msg)

void	finishCUDA (LocalWorkMsg *msg)

void	finishCUDAP2 (LocalWorkMsg *msg)

void	finishCUDAP3 (LocalWorkMsg *msg)

void	enqueueMIC (LocalWorkMsg *msg)

void	finishMIC (LocalWorkMsg *msg)

void	enqueueLCPO (LocalWorkMsg *msg)

void	mapComputes (void)

void	sendPatchMap (void)

void	sendComputeMap (void)

void	saveComputeMapChanges (int, CkGroupID)

void	recvComputeMapChanges (ComputeMapChangeMsg *)

void	doneSaveComputeMap (CkReductionMsg *)

FullAtomList *	createAtomLists (const char *basename=0)

void	createHomePatches (void)

void	distributeHomePatches (void)

void	reinitAtoms (const char *basename=0)

void	patchMapInit (void)

void	assignNodeToPatch (void)

void	savePatchMap (PatchMapMsg *msg)

void	setPatchMapArrived (bool s)

void	initHostDeviceLDB ()

void	contributeHostDeviceLDB (int peSetLen, int *peSet)

void	setDeviceLDBParams (int dt, int hs, int sp1, int pp1, int pp2)

Static Public Member Functions
static void	messageEnqueueWork (Compute *)

static void	messageFinishCUDA (Compute *)

static void	messageFinishMIC (Compute *)

static void	sortPmePes (int *pmepes, int xdim, int ydim)

static void	peOrderingReady ()

static void	send_initHostDeviceLDB ()

static void	send_contributeHostDeviceLDB (int peSetLen, int *peSet)

static void	send_setDeviceLDBParams (int dt, int hs, int sp1, int pp1, int pp2)

static void	buildNodeAwarePeOrdering (void)

Static Public Attributes
static int	peOrderingInit

static int *	peDiffuseOrdering

static int *	peDiffuseOrderingIndex

static int *	peCompactOrdering

static int *	peCompactOrderingIndex

Detailed Description

Definition at line 42 of file WorkDistrib.h.

Constructor & Destructor Documentation

◆ WorkDistrib()

WorkDistrib::WorkDistrib ( )

Definition at line 108 of file WorkDistrib.C.

References eventMachineProgress, and NAMD_bug().

 {
   CkpvAccess(BOCclass_group).workDistrib = thisgroup;
   patchMapArrived = false;
   computeMapArrived = false;
 
 #if CMK_SMP
 #define MACHINE_PROGRESS
 #else
 #define MACHINE_PROGRESS { traceUserEvent(eventMachineProgress);  CmiMachineProgressImpl(); }
   if ( CkMyNodeSize() > 1 ) NAMD_bug("CkMyNodeSize() > 1 for non-smp build");
   eventMachineProgress = traceRegisterUserEvent("CmiMachineProgressImpl",233);
 #endif
 }

◆ ~WorkDistrib()

WorkDistrib::~WorkDistrib ( void )

Definition at line 124 of file WorkDistrib.C.

125 { }

Member Function Documentation

◆ assignNodeToPatch()

void WorkDistrib::assignNodeToPatch ( void )

Definition at line 1456 of file WorkDistrib.C.

References endi(), Patch::getNumAtoms(), iINFO(), iout, Node::molecule, NAMD_die(), PatchMap::node(), Molecule::numAtoms, Node::numNodes(), PatchMap::numPatches(), PatchMap::Object(), Node::Object(), PatchMap::patch(), Node::simParameters, SimParameters::simulatedPEs, and SimParameters::simulateInitialMapping.

Referenced by Node::startup().

 {
   PatchMap *patchMap = PatchMap::Object();
   int nNodes = Node::Object()->numNodes();
   SimParameters *simparam = Node::Object()->simParameters;
   if(simparam->simulateInitialMapping) {
           nNodes = simparam->simulatedPEs;
   }
 
 #if (CMK_BLUEGENEP | CMK_BLUEGENEL) && USE_TOPOMAP 
   TopoManager tmgr;
   int numPes = tmgr.getDimNX() * tmgr.getDimNY() * tmgr.getDimNZ();
   if (numPes > patchMap->numPatches() && (assignPatchesTopoGridRecBisection() > 0)) {
     CkPrintf ("Blue Gene/L topology partitioner finished successfully \n");
   }
   else
 #endif
   assignPatchesSpaceFillingCurve();       
   
   int *nAtoms = new int[nNodes];
   int numAtoms=0;
   int i;
   for(i=0; i < nNodes; i++)
     nAtoms[i] = 0;
 
   for(i=0; i < patchMap->numPatches(); i++)
   {
     //    iout << iINFO << "Patch " << i << " has " 
     //   << patchMap->patch(i)->getNumAtoms() << " atoms and "
     //   << patchMap->patch(i)->getNumAtoms() * 
     //            patchMap->patch(i)->getNumAtoms() 
     //   << " pairs.\n" << endi;         
 #ifdef MEM_OPT_VERSION
       numAtoms += patchMap->numAtoms(i);
       nAtoms[patchMap->node(i)] += patchMap->numAtoms(i);         
 #else
     if (patchMap->patch(i)) {
       numAtoms += patchMap->patch(i)->getNumAtoms();
       nAtoms[patchMap->node(i)] += patchMap->patch(i)->getNumAtoms();     
     }
 #endif
   }
 
   if ( numAtoms != Node::Object()->molecule->numAtoms ) {
     for(i=0; i < nNodes; i++)
       iout << iINFO << nAtoms[i] << " atoms assigned to node " << i << "\n" << endi;
     iout << iINFO << "Assigned " << numAtoms << " atoms but expected " << Node::Object()->molecule->numAtoms << "\n" << endi;
     NAMD_die("Incorrect atom count in WorkDistrib::assignNodeToPatch\n");
   }
 
   delete [] nAtoms;
  
   //  PatchMap::Object()->printPatchMap();
 }

◆ buildNodeAwarePeOrdering()

void WorkDistrib::buildNodeAwarePeOrdering ( void )

static

Definition at line 183 of file WorkDistrib.C.

References ResizeArray< Elem >::begin(), endi(), iout, iWARN(), peCompactOrdering, peCompactOrderingIndex, peDiffuseOrdering, peDiffuseOrderingIndex, peOrderingInit, peOrderingReady(), randtopo, and Random::reorder().

Referenced by build_ordering().

                                            {
 
  CmiMemLock();
  if ( ! peOrderingInit ) {
   //CkPrintf("WorkDistrib::buildNodeAwarePeOrdering on pe %d\n", CkMyPe());
 
   const int numPhys = CmiNumPhysicalNodes();
   const int numNode = CmiNumNodes();
   const int numPe = CmiNumPes();
   ResizeArray<int> numNodeInPhys(numPhys);
   ResizeArray<int> rankInPhysOfNode(numNode);
 
   peDiffuseOrdering = new int[numPe];
   peDiffuseOrderingIndex = new int[numPe];
   peCompactOrdering = new int[numPe];
   peCompactOrderingIndex = new int[numPe];
 
   int k = 0;
   for ( int ph=0; ph<numPhys; ++ph ) {
     int *pes, npes;
     CmiGetPesOnPhysicalNode(ph, &pes, &npes);
     for ( int i=0; i<npes; ++i, ++k ) {
       peCompactOrdering[k] = pes[i];
     }
     numNodeInPhys[ph] = 0;
     for ( int i=0, j=0; i<npes; i += CmiNodeSize(CmiNodeOf(pes[i])), ++j ) {
       rankInPhysOfNode[CmiNodeOf(pes[i])] = j;
       numNodeInPhys[ph] += 1;
     }
   }
 
   if ( randtopo && numPhys > 2 ) {
     if ( ! CkMyNode() ) {
       iout << iWARN << "RANDOMIZING PHYSICAL NODE ORDERING\n" << endi;
     }
     ResizeArray<int> randPhysOrder(numPhys);
     for ( int j=0; j<numPhys; ++j ) {
       randPhysOrder[j] = j;
     }
     Random(314159265).reorder(randPhysOrder.begin()+2, numPhys-2);
     for ( int j=0, k=0; j<numPhys; ++j ) {
       const int ph = randPhysOrder[j];
       int *pes, npes;
       CmiGetPesOnPhysicalNode(ph, &pes, &npes);
       for ( int i=0; i<npes; ++i, ++k ) {
         peCompactOrdering[k] = pes[i];
       }
     }
   }
 
   for ( int i=0; i<numPe; ++i ) {
     peDiffuseOrdering[i] = i;
   }
   std::sort(peDiffuseOrdering, peDiffuseOrdering+numPe,
     pe_sortop_bit_reversed(rankInPhysOfNode.begin()));
 
   for ( int i=0; i<numPe; ++i ) {
     peDiffuseOrderingIndex[peDiffuseOrdering[i]] = i;
     peCompactOrderingIndex[peCompactOrdering[i]] = i;
   }
 
   if ( 0 && CmiMyNode() == 0 ) for ( int i=0; i<numPe; ++i ) {
     CkPrintf("order %5d %5d %5d %5d %5d\n", i,
       peDiffuseOrdering[i],
       peDiffuseOrderingIndex[i],
       peCompactOrdering[i],
       peCompactOrderingIndex[i]);
   }
 
   peOrderingInit = 1;
  }
  CmiMemUnlock();
  peOrderingReady();
 
 }

◆ contributeHostDeviceLDB()

void WorkDistrib::contributeHostDeviceLDB	(	int	peSetLen,
		int *	peSet
	)

Definition at line 3539 of file WorkDistrib.C.

                                                                    {
   #if defined(NAMD_MIC)
     mic_contributeHostDeviceLDB(peSetLen, peSet);
   #endif
 }

◆ createAtomLists()

FullAtomList * WorkDistrib::createAtomLists ( const char * basename = 0 )

Definition at line 654 of file WorkDistrib.C.

References ResizeArray< Elem >::add(), SimParameters::alchOn, Lattice::apply_transform(), PatchMap::assignToPatch(), Molecule::atomcharge(), CompAtomExt::atomFixed, HydrogenGroupID::atomID, Molecule::atommass(), HydrogenGroupID::atomsInGroup, HydrogenGroupID::atomsInMigrationGroup, Molecule::atomvdwtype(), ResizeArray< Elem >::begin(), SimParameters::binaryOutput, CompAtom::charge, SimParameters::comMove, Node::configList, StringList::data, CompAtomExt::dispcoef, endi(), FALSE, ConfigList::find(), SimParameters::fixedAtomsOn, FullAtom::fixedPosition, PDB::get_all_positions(), Molecule::get_fep_type(), Parameters::get_vdw_params(), Molecule::getAtoms(), getWaterModelGroupSize(), PatchMap::gridsize_a(), PatchMap::gridsize_b(), PatchMap::gridsize_c(), CompAtomExt::groupFixed, Molecule::hydrogenGroup, CompAtom::hydrogenGroupSize, CompAtomExt::id, PatchMap::index_a(), PatchMap::index_b(), PatchMap::index_c(), SimParameters::initialTemp, iout, Molecule::is_atom_fixed(), HydrogenGroupID::isGP, HydrogenGroupID::isMP, CompAtom::isWater, Molecule::langevin_param(), FullAtom::langevinParam, SimParameters::lattice, SimParameters::lesOn, SimParameters::LJPMEOn, FullAtom::mass, PatchMap::max_a(), PatchMap::max_b(), PatchMap::max_c(), FullAtom::migrationGroupSize, PatchMap::min_a(), PatchMap::min_b(), PatchMap::min_c(), Node::molecule, NAMD_bug(), NAMD_die(), Lattice::nearest(), CompAtom::nonbondedGroupSize, SimParameters::nonbondedScaling, PDB::num_atoms(), PatchMap::numPatches(), PatchMap::Object(), order, SimParameters::outputPatchDetails, SimParameters::pairInteractionOn, CompAtom::partition, Node::pdb, CompAtom::position, SimParameters::pressureProfileAtomTypes, read_binary_file(), FullAtom::recipMass, Molecule::rigid_bond_length(), RIGID_NONE, FullAtom::rigidBondLength, SimParameters::rigidBonds, Node::simParameters, ResizeArray< Elem >::size(), sortAtomsForPatches(), SimParameters::staticAtomAssignment, atom_constants::status, FullAtom::status, FullAtom::transform, TRUE, SimParameters::useSettle, CompAtom::vdwType, FullAtom::velocity, and SimParameters::watmodel.

Referenced by createHomePatches(), and reinitAtoms().

 {
   int i;
   StringList *current;  //  Pointer used to retrieve configuration items
   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
   Node *node = nd.ckLocalBranch();
   PatchMap *patchMap = PatchMap::Object();
   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
   PatchMgr *patchMgr = pm.ckLocalBranch();
   SimParameters *params = node->simParameters;
   Molecule *molecule = node->molecule;
   PDB *pdb = node->pdb;
 
   int numPatches = patchMap->numPatches();
   int numAtoms = pdb->num_atoms();
 
   Vector *positions = new Position[numAtoms];
   Vector *velocities = new Velocity[numAtoms];
 
  if ( basename ) {
   if ( params->binaryOutput ) {
     read_binary_file((std::string(basename)+".coor").c_str(), positions, numAtoms);
     read_binary_file((std::string(basename)+".vel").c_str(), velocities, numAtoms);
   } else {
     PDB coorpdb((std::string(basename)+".coor").c_str());
     if ( coorpdb.num_atoms() != numAtoms ) {
       NAMD_die("Incorrect atom count in coordinate pdb file");
     }
     coorpdb.get_all_positions(positions);
     velocities_from_PDB((std::string(basename)+".vel").c_str(), velocities, numAtoms);
   }
  } else {
   pdb->get_all_positions(positions);
 
   if ( params->initialTemp < 0.0 ) {
     Bool binvels=FALSE;
 
     //  Reading the veolcities from a PDB
     current = node->configList->find("velocities");
 
     if (current == NULL) {
       current = node->configList->find("binvelocities");
       binvels = TRUE;
     }
 
     if (!binvels) {
       velocities_from_PDB(current->data, velocities, numAtoms);
     }
     else {
       velocities_from_binfile(current->data, velocities, numAtoms);
     }
   }
   else {
     // Random velocities for a given temperature
     random_velocities(params->initialTemp, molecule, velocities, numAtoms);
   }
  }
 
   //  If COMMotion == no, remove center of mass motion
   if (!(params->comMove)) {
     remove_com_motion(velocities, molecule, numAtoms);
   }
 
   FullAtomList *atoms = new FullAtomList[numPatches];
 
   const Lattice lattice = params->lattice;
 
     if ( params->staticAtomAssignment ) {
       FullAtomList sortAtoms;
       for ( i=0; i < numAtoms; i++ ) {
         HydrogenGroupID &h = molecule->hydrogenGroup[i];
         if ( ! h.isMP ) continue;
         FullAtom a;
         a.id = i;
         a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
         a.position = positions[h.atomID];
         sortAtoms.add(a);
       } 
       int *order = new int[sortAtoms.size()];
       for ( i=0; i < sortAtoms.size(); i++ ) {
         order[i] = i;
       } 
       int *breaks = new int[numPatches];
       sortAtomsForPatches(order,breaks,sortAtoms.begin(),
                         sortAtoms.size(),numAtoms,
                         patchMap->gridsize_c(),
                         patchMap->gridsize_b(),
                         patchMap->gridsize_a());
 
       i = 0;
       for ( int pid = 0; pid < numPatches; ++pid ) {
         int iend = breaks[pid];
         for ( ; i<iend; ++i ) {
           FullAtom &sa = sortAtoms[order[i]];
           int mgs = sa.migrationGroupSize;
 /*
 CkPrintf("patch %d (%d %d %d) has group %d atom %d size %d at %.2f %.2f %.2f\n",
           pid, patchMap->index_a(pid), patchMap->index_b(pid), 
           patchMap->index_c(pid), order[i], sa.id, mgs,
           sa.position.x, sa.position.y, sa.position.z);
 */
           for ( int k=0; k<mgs; ++k ) {
             HydrogenGroupID &h = molecule->hydrogenGroup[sa.id + k];
             int aid = h.atomID;
             FullAtom a;
             a.id = aid;
             a.position = positions[aid];
             a.velocity = velocities[aid];
             a.vdwType = molecule->atomvdwtype(aid);
             a.status = molecule->getAtoms()[aid].status;
             a.langevinParam = molecule->langevin_param(aid);
             a.hydrogenGroupSize = h.isGP ? h.atomsInGroup : 0;
             a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
             if(params->rigidBonds != RIGID_NONE) {
               a.rigidBondLength = molecule->rigid_bond_length(aid);
             }else{
               a.rigidBondLength = 0.0;
             }
             atoms[pid].add(a);
           }
         }
 CkPrintf("patch %d (%d %d %d) has %d atoms\n",
           pid, patchMap->index_a(pid), patchMap->index_b(pid), 
           patchMap->index_c(pid), atoms[pid].size());
       }
       delete [] order;
       delete [] breaks;
     } else
     {
     // split atoms into patches based on migration group and position
     int aid, pid=0;
     for(i=0; i < numAtoms; i++)
       {
       // Assign atoms to patches without splitting hydrogen groups.
       // We know that the hydrogenGroup array is sorted with group parents
       // listed first.  Thus, only change the pid if an atom is a group parent.
       HydrogenGroupID &h = molecule->hydrogenGroup[i];
       aid = h.atomID;
       FullAtom a;
       a.id = aid;
       a.position = positions[aid];
       a.velocity = velocities[aid];
       a.vdwType = molecule->atomvdwtype(aid);
       a.status = molecule->getAtoms()[aid].status;
       a.langevinParam = molecule->langevin_param(aid);
       a.hydrogenGroupSize = h.isGP ? h.atomsInGroup : 0;
       a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
       if(params->rigidBonds != RIGID_NONE) {
         a.rigidBondLength = molecule->rigid_bond_length(aid);
       }else{
         a.rigidBondLength = 0.0;
       }
       if (h.isMP) {
         pid = patchMap->assignToPatch(positions[aid],lattice);
       } // else: don't change pid
       atoms[pid].add(a);
       }
     }
 
   delete [] positions;
   delete [] velocities;
 
   for(i=0; i < numPatches; i++)
   {
     ScaledPosition center(0.5*(patchMap->min_a(i)+patchMap->max_a(i)),
                           0.5*(patchMap->min_b(i)+patchMap->max_b(i)),
                           0.5*(patchMap->min_c(i)+patchMap->max_c(i)));
 
     int n = atoms[i].size();
     FullAtom *a = atoms[i].begin();
     int j;
 //Modifications for alchemical fep
     Bool alchOn = params->alchOn;
 //fepe
     Bool lesOn = params->lesOn;
   
     Bool pairInteractionOn = params->pairInteractionOn;
 
     Bool pressureProfileTypes = (params->pressureProfileAtomTypes > 1);
 
     Transform mother_transform;
     for(j=0; j < n; j++)
     {
       int aid = a[j].id;
 
       a[j].nonbondedGroupSize = 0;  // must be set based on coordinates
 
       a[j].atomFixed = molecule->is_atom_fixed(aid) ? 1 : 0;
       a[j].fixedPosition = a[j].position;
 
       if ( a[j].migrationGroupSize ) {
        if ( a[j].migrationGroupSize != a[j].hydrogenGroupSize ) {
             Position pos = a[j].position;
             int mgs = a[j].migrationGroupSize;
             int c = 1;
             for ( int k=a[j].hydrogenGroupSize; k<mgs;
                                 k+=a[j+k].hydrogenGroupSize ) {
               pos += a[j+k].position;
               ++c;
             }
             pos *= 1./c;
             mother_transform = a[j].transform;  // should be 0,0,0
             pos = lattice.nearest(pos,center,&mother_transform);
             a[j].position = lattice.apply_transform(a[j].position,mother_transform);
             a[j].transform = mother_transform;
        } else {
         a[j].position = lattice.nearest(
                 a[j].position, center, &(a[j].transform));
         mother_transform = a[j].transform;
        }
       } else {
         a[j].position = lattice.apply_transform(a[j].position,mother_transform);
         a[j].transform = mother_transform;
       }
 
       a[j].mass = molecule->atommass(aid);
       // Using double precision division for reciprocal mass.
       a[j].recipMass = ( a[j].mass > 0 ? (1. / a[j].mass) : 0 );
       a[j].charge = molecule->atomcharge(aid);
       if ( params->LJPMEOn ) {
         const int index = a[j].vdwType;
         const float scaling = params->nonbondedScaling;
         float sigma, epsilon, sigma14, epsilon14;
         molecule->params->get_vdw_params(&sigma, &epsilon, &sigma14, &epsilon14, index);
         a[j].dispcoef = 2*sigma*sigma*sigma*sqrt(scaling * epsilon);
       }
       else {
         a[j].dispcoef = 0;
       }
 
 //Modifications for alchemical fep
       if ( alchOn || lesOn || pairInteractionOn || pressureProfileTypes) {
         a[j].partition = molecule->get_fep_type(aid);
       } 
       else {
         a[j].partition = 0;
       }
 //fepe
 
     }
 
 #if 0
     int size, allfixed, k;
     for(j=0; j < n; j+=size) {
       size = a[j].hydrogenGroupSize;
       if ( ! size ) {
         NAMD_bug("Mother atom with hydrogenGroupSize of 0!");
       }
       allfixed = 1;
       for ( k = 0; k < size; ++k ) {
         allfixed = ( allfixed && (a[j+k].atomFixed) );
       }
       for ( k = 0; k < size; ++k ) {
         a[j+k].groupFixed = allfixed ? 1 : 0;
       }
       // DH - set isWater flag
       // based on same condition as used for determining Settle:
       // The rigidBondLength of the FIRST water atom is set greater than 0.
       if (a[j].rigidBondLength > 0) {
         for (k = 0;  k < size;  k++) {
           a[j+k].isWater = 1;
         }
       }
     }
 #else
     // DH - promote water checking to take place before simulation
     // Flag the Settle water molecules.
     int size, allfixed;
     const WaterModel watmodel = params->watmodel;
     const int wathgsize = getWaterModelGroupSize(watmodel);
     const int fixedAtomsOn = params->fixedAtomsOn;
     const int useSettle = params->useSettle;
     for(int j=0; j < n; j+=size) {
       size = a[j].hydrogenGroupSize;
       if ( ! size ) {
         NAMD_bug("Mother atom with hydrogenGroupSize of 0!");
       }
       allfixed = 1;
       for (int k = 0; k < size; ++k ) {
         allfixed = ( allfixed && (a[j+k].atomFixed) );
       }
       for (int k = 0; k < size; ++k ) {
         a[j+k].groupFixed = allfixed ? 1 : 0;
       }
       // DH - Set isWater flag for SETTLE water molecules,
       // based on same condition as used in HomePatch::buildRattleList():
       // The rigidBondLength of the FIRST water atom is set greater than 0.
       if (a[j].rigidBondLength > 0) {
         if (size != wathgsize) {
           char errmsg[256];
           sprintf(errmsg,
               "Water molecule starting with atom %d contains %d atoms "
               "but the specified water model requires %d atoms.\n",
               a[j].id+1, size, wathgsize
               );
           NAMD_die(errmsg);
         }
         int anyfixed = 0;
         for (int k = 0;  k < size;  k++) {
           anyfixed += ( fixedAtomsOn && a[j+k].atomFixed );
         }
         if (useSettle && !anyfixed) {
           for (int k = 0;  k < size;  k++) {
             a[j+k].isWater = 1;
           }
         }
       }
     }
 #endif
 
     if ( params->outputPatchDetails ) {
       int patchId = i;
       int numAtomsInPatch = n;
       int numFixedAtomsInPatch = 0;
       int numAtomsInFixedGroupsInPatch = 0;
       for(j=0; j < n; j++) {
         numFixedAtomsInPatch += ( a[j].atomFixed ? 1 : 0 );
         numAtomsInFixedGroupsInPatch += ( a[j].groupFixed ? 1 : 0 );
       }
       iout << "PATCH_DETAILS:"
            << " patch " << patchId
            << " atoms " << numAtomsInPatch
            << " fixed_atoms " << numFixedAtomsInPatch
            << " fixed_groups " << numAtomsInFixedGroupsInPatch
            << "\n" << endi;
     }
   }
 
   return atoms;
 
 }

◆ createHomePatches()

void WorkDistrib::createHomePatches ( void )

Definition at line 989 of file WorkDistrib.C.

References createAtomLists(), PatchMgr::createHomePatch(), DebugM, endi(), iINFO(), iout, PatchMap::numPatches(), PatchMap::Object(), and ResizeArray< Elem >::size().

Referenced by Node::startup().

 {
   int i;
   PatchMap *patchMap = PatchMap::Object();
   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
   PatchMgr *patchMgr = pm.ckLocalBranch();
 
   int numPatches = patchMap->numPatches();
 
   FullAtomList *atoms = createAtomLists();
     
 #ifdef MEM_OPT_VERSION
 /*  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
   Node *node = nd.ckLocalBranch();
   node->molecule->delEachAtomSigs();
   node->molecule->delMassChargeSpace();
 */
 #endif
 
   int maxAtoms = -1;
   int maxPatch = -1;
   for(i=0; i < numPatches; i++) {
     int numAtoms = atoms[i].size();
     if ( numAtoms > maxAtoms ) { maxAtoms = numAtoms; maxPatch = i; }
   }
   iout << iINFO << "LARGEST PATCH (" << maxPatch <<
         ") HAS " << maxAtoms << " ATOMS\n" << endi;
 
 #ifdef SHOW_HISTOGRAM_HGROUP_SIZES
   // histogram hydrogen group sizes
   int hgroupsize[9] = { 0 };
   int numhgroups = 0;
   int numwaters = 0;
   int maxhgroupsize = 0;
   for (i = 0;  i < numPatches;  i++) {
     const FullAtomList& a = atoms[i];
     int numAtoms = a.size();
     int hgs = 1;  // init to something sane
     for (int j = 0;  j < numAtoms;  j += hgs) {
       hgs = a[j].hydrogenGroupSize;
       int histndx = ( hgs > 8 ? 8 : hgs );
       hgroupsize[ histndx ]++;
       numhgroups++;
       if (a[j].isWater) numwaters++;
       if (maxhgroupsize < hgs) maxhgroupsize = hgs;
     }
   }
   int hgslast = ( maxhgroupsize > 8 ? 8 : maxhgroupsize );
   printf("Number of hydrogen groups:           %7d\n", numhgroups);
   printf("Number of settle water molecules:    %7d\n", numwaters);
   printf("Number of remaining hydrogen groups: %7d\n", numhgroups - numwaters);
   printf("Largest hydrogen group size:         %7d\n", maxhgroupsize);
   printf("Histogram of hydrogen group sizes:\n");
   int hgstotal = 0;
   for (i = 0;  i <= hgslast;  i++) {
     printf("     size %d     count %d\n", i, hgroupsize[i]);
     hgstotal += hgroupsize[i];
   }
   printf("Checksum over hydrogen group sizes:  %7d\n", hgstotal);
 #endif
 
   for(i=0; i < numPatches; i++)
   {
     if ( ! ( i % 100 ) )
     {
       DebugM(3,"Created " << i << " patches so far.\n");
     }
 
     patchMgr->createHomePatch(i,atoms[i]);
   }
 
   delete [] atoms;
 }

◆ distributeHomePatches()

void WorkDistrib::distributeHomePatches ( void )

Definition at line 1063 of file WorkDistrib.C.

References DebugM, PatchMgr::movePatch(), Node::myid(), PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), and PatchMgr::sendMovePatches().

Referenced by Node::startup().

                                         {
   // ref BOC
   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
   Node *node = nd.ckLocalBranch();
   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
   PatchMgr *patchMgr = pm.ckLocalBranch();
   // ref singleton
   PatchMap *patchMap = PatchMap::Object();
 
   // Move patches to the proper node
   for(int i=0;i < patchMap->numPatches(); i++)
   {
     if (patchMap->node(i) != node->myid() )
     {
       DebugM(3,"patchMgr->movePatch("
         << i << "," << patchMap->node(i) << ")\n");
       patchMgr->movePatch(i,patchMap->node(i));
     }
   }
   patchMgr->sendMovePatches();
 }

◆ doneSaveComputeMap()

void WorkDistrib::doneSaveComputeMap ( CkReductionMsg * msg )

Definition at line 430 of file WorkDistrib.C.

                                                         {
   delete msg;
 
   CkSendMsgBranch(saveComputeMapReturnEP, CkAllocMsg(0,0,0), 0, saveComputeMapReturnChareID);
 }

◆ enqueueAngles()

void WorkDistrib::enqueueAngles ( LocalWorkMsg * msg )

Definition at line 3110 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueAniso()

void WorkDistrib::enqueueAniso ( LocalWorkMsg * msg )

Definition at line 3134 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueBonds()

void WorkDistrib::enqueueBonds ( LocalWorkMsg * msg )

Definition at line 3104 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueCrossterms()

void WorkDistrib::enqueueCrossterms ( LocalWorkMsg * msg )

Definition at line 3140 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                      {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueCUDA()

void WorkDistrib::enqueueCUDA ( LocalWorkMsg * msg )

Definition at line 3244 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ enqueueCUDAP2()

void WorkDistrib::enqueueCUDAP2 ( LocalWorkMsg * msg )

Definition at line 3247 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ enqueueCUDAP3()

void WorkDistrib::enqueueCUDAP3 ( LocalWorkMsg * msg )

Definition at line 3250 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ enqueueDihedrals()

void WorkDistrib::enqueueDihedrals ( LocalWorkMsg * msg )

Definition at line 3116 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                     {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueExcls()

void WorkDistrib::enqueueExcls ( LocalWorkMsg * msg )

Definition at line 3098 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueGromacsPair()

void WorkDistrib::enqueueGromacsPair ( LocalWorkMsg * msg )

Definition at line 3153 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                       {
   msg->compute->doWork();
   MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("\nWorkDistrib LocalWorkMsg recycling failed! Check enqueueGromacsPair from WorkDistrib.C\n");
 }

◆ enqueueImpropers()

void WorkDistrib::enqueueImpropers ( LocalWorkMsg * msg )

Definition at line 3122 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                     {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueLCPO()

void WorkDistrib::enqueueLCPO ( LocalWorkMsg * msg )

Definition at line 3167 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, and NAMD_bug().

                                                {
   msg->compute->doWork();
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueMIC()

void WorkDistrib::enqueueMIC ( LocalWorkMsg * msg )

Definition at line 3268 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                               {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ enqueueOneFourNbThole()

void WorkDistrib::enqueueOneFourNbThole ( LocalWorkMsg * msg )

Definition at line 3146 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                          {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueuePme()

void WorkDistrib::enqueuePme ( LocalWorkMsg * msg )

Definition at line 3161 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                               {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueSelfA1()

void WorkDistrib::enqueueSelfA1 ( LocalWorkMsg * msg )

Definition at line 3172 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueSelfA2()

void WorkDistrib::enqueueSelfA2 ( LocalWorkMsg * msg )

Definition at line 3177 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueSelfA3()

void WorkDistrib::enqueueSelfA3 ( LocalWorkMsg * msg )

Definition at line 3182 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueSelfB1()

void WorkDistrib::enqueueSelfB1 ( LocalWorkMsg * msg )

Definition at line 3188 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueSelfB2()

void WorkDistrib::enqueueSelfB2 ( LocalWorkMsg * msg )

Definition at line 3193 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueSelfB3()

void WorkDistrib::enqueueSelfB3 ( LocalWorkMsg * msg )

Definition at line 3198 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueThole()

void WorkDistrib::enqueueThole ( LocalWorkMsg * msg )

Definition at line 3128 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWork()

void WorkDistrib::enqueueWork ( LocalWorkMsg * msg )

Definition at line 3092 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWorkA1()

void WorkDistrib::enqueueWorkA1 ( LocalWorkMsg * msg )

Definition at line 3204 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWorkA2()

void WorkDistrib::enqueueWorkA2 ( LocalWorkMsg * msg )

Definition at line 3209 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWorkA3()

void WorkDistrib::enqueueWorkA3 ( LocalWorkMsg * msg )

Definition at line 3214 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWorkB1()

void WorkDistrib::enqueueWorkB1 ( LocalWorkMsg * msg )

Definition at line 3220 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWorkB2()

void WorkDistrib::enqueueWorkB2 ( LocalWorkMsg * msg )

Definition at line 3225 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWorkB3()

void WorkDistrib::enqueueWorkB3 ( LocalWorkMsg * msg )

Definition at line 3230 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                  {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ enqueueWorkC()

void WorkDistrib::enqueueWorkC ( LocalWorkMsg * msg )

Definition at line 3238 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
   msg->compute->doWork();  MACHINE_PROGRESS
   if ( msg->compute->localWorkMsg != msg )
     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
 }

◆ finishCUDA()

void WorkDistrib::finishCUDA ( LocalWorkMsg * msg )

Definition at line 3258 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                               {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ finishCUDAP2()

void WorkDistrib::finishCUDAP2 ( LocalWorkMsg * msg )

Definition at line 3261 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                 {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ finishCUDAP3()

void WorkDistrib::finishCUDAP3 ( LocalWorkMsg * msg )

Definition at line 3264 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                 {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ finishCUDAPatch()

void WorkDistrib::finishCUDAPatch ( FinishWorkMsg * msg )

Definition at line 3254 of file WorkDistrib.C.

References FinishWorkMsg::compute, FinishWorkMsg::data, and Compute::finishPatch().

                                                     {
   msg->compute->finishPatch(msg->data);
 }

◆ finishMIC()

void WorkDistrib::finishMIC ( LocalWorkMsg * msg )

Definition at line 3271 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                              {
   msg->compute->doWork();  MACHINE_PROGRESS
 }

◆ initHostDeviceLDB()

void WorkDistrib::initHostDeviceLDB ( )

Definition at line 3526 of file WorkDistrib.C.

                                     {
   #if defined(NAMD_MIC)
     mic_hostDeviceLDB();
   #endif
 }

◆ mapComputes()

void WorkDistrib::mapComputes ( void )

Definition at line 2407 of file WorkDistrib.C.

References ComputeMap::allocateCids(), SimParameters::bondedCUDA, computeAnglesType, computeAnisoType, computeBondsType, computeConsForceType, computeConsTorqueType, computeCrosstermsType, computeCylindricalBCType, computeDihedralsType, computeEFieldType, computeEwaldType, computeExclsType, computeExtType, computeFmmType, computeFullDirectType, computeGBISserType, computeGlobalType, computeGridForceType, computeGromacsPairType, computeImpropersType, computeLjPmeSerialType, computeMsmMsaType, computeMsmSerialType, computeMsmType, computeNonbondedCUDA2Type, computeNonbondedMICType, computeOneFourNbTholeType, computePmeCUDAType, computePmeType, computeQMType, computeRestraintsType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfGromacsPairType, computeSelfImpropersType, computeSelfOneFourNbTholeType, computeSelfTholeType, computeSphericalBCType, computeStirType, computeTclBCType, computeTholeType, SimParameters::consForceOn, SimParameters::consTorqueOn, SimParameters::constraintsOn, SimParameters::cylindricalBCOn, DebugM, SimParameters::drudeOn, SimParameters::eFieldOn, SimParameters::extForcesOn, SimParameters::FMAOn, SimParameters::FMMOn, SimParameters::fullDirectOn, SimParameters::GBISserOn, SimParameters::globalForcesOn, SimParameters::goGroPair, SimParameters::LCPOOn, SimParameters::LJPMEOn, ComputeMap::loadComputeMap(), SimParameters::mgridforceOn, SimParameters::MSMOn, SimParameters::MsmSerialOn, NAMD_die(), PatchMap::Object(), Node::Object(), ComputeMap::Object(), SimParameters::pairInteractionOn, SimParameters::pairInteractionSelf, SimParameters::PMEOn, SimParameters::pressureProfileEwaldOn, SimParameters::qmForcesOn, ComputeMap::saveComputeMap(), Node::simParameters, simParams, SimParameters::sphericalBCOn, SimParameters::stirOn, SimParameters::tclBCOn, SimParameters::useDPME, and SimParameters::usePMECUDA.

Referenced by Node::startup().

 {
   PatchMap *patchMap = PatchMap::Object();
   ComputeMap *computeMap = ComputeMap::Object();
   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
   Node *node = nd.ckLocalBranch();
 
   DebugM(3,"Mapping computes\n");
 
   computeMap->allocateCids();
 
   // Handle full electrostatics
   if ( node->simParameters->fullDirectOn )
     mapComputeHomePatches(computeFullDirectType);
   if ( node->simParameters->FMAOn )
 #ifdef DPMTA
     mapComputeHomePatches(computeDPMTAType);
 #else
     NAMD_die("This binary does not include DPMTA (FMA).");
 #endif
   if ( node->simParameters->PMEOn ) {
 #ifdef DPME
     if ( node->simParameters->useDPME )
       mapComputeHomePatches(computeDPMEType);
     else {
       mapComputeHomePatches(computePmeType);
       if ( node->simParameters->pressureProfileEwaldOn )
         mapComputeHomePatches(computeEwaldType);
     }
 #else
 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
     if (node->simParameters->usePMECUDA) {
       mapComputePatch(computePmeCUDAType);
     } else
 #endif
     {
       mapComputePatch(computePmeType);
     }
     if ( node->simParameters->pressureProfileEwaldOn )
       mapComputeHomePatches(computeEwaldType);
 #endif
   }
 
   if ( node->simParameters->globalForcesOn ) {
     DebugM(2,"adding ComputeGlobal\n");
     mapComputeHomePatches(computeGlobalType);
   }
 
   if ( node->simParameters->extForcesOn )
     mapComputeHomePatches(computeExtType);
 
   if ( node->simParameters->qmForcesOn )
     mapComputeHomePatches(computeQMType);
 
   if ( node->simParameters->GBISserOn )
     mapComputeHomePatches(computeGBISserType);
 
   if ( node->simParameters->MsmSerialOn )
     mapComputeHomePatches(computeMsmSerialType);
 
   if ( node->simParameters->LJPMEOn )
     mapComputeHomePatches(computeLjPmeSerialType);
 #ifdef CHARM_HAS_MSA
   else if ( node->simParameters->MSMOn )
     mapComputeHomePatches(computeMsmMsaType);
 #else
   else if ( node->simParameters->MSMOn )
     mapComputeHomePatches(computeMsmType);
 #endif
 
   if ( node->simParameters->FMMOn )
     mapComputeHomePatches(computeFmmType);
 
 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
 #ifdef BONDED_CUDA
   if (node->simParameters->bondedCUDA) {
     mapComputeNode(computeBondedCUDAType);
   }
 #endif
 #endif
 
 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
   mapComputeNode(computeNonbondedCUDA2Type);
   mapComputeHomeTuples(computeExclsType);
   mapComputePatch(computeSelfExclsType);
 #endif
 
 #ifdef NAMD_MIC
   mapComputeNode(computeNonbondedMICType);
 #endif
 
   mapComputeNonbonded();
 
   if ( node->simParameters->LCPOOn ) {
     mapComputeLCPO();
   }
 
   // If we're doing true pair interactions, no need for bonded terms.
   // But if we're doing within-group interactions, we do need them.
   if ( !node->simParameters->pairInteractionOn || 
       node->simParameters->pairInteractionSelf) { 
     mapComputeHomeTuples(computeBondsType);
     mapComputeHomeTuples(computeAnglesType);
     mapComputeHomeTuples(computeDihedralsType);
     mapComputeHomeTuples(computeImpropersType);
     mapComputeHomeTuples(computeCrosstermsType);
     mapComputePatch(computeSelfBondsType);
     mapComputePatch(computeSelfAnglesType);
     mapComputePatch(computeSelfDihedralsType);
     mapComputePatch(computeSelfImpropersType);
     mapComputePatch(computeSelfCrosstermsType);
   }
 
   if ( node->simParameters->goGroPair ) {
       // JLai
       mapComputeHomeTuples(computeGromacsPairType);
       mapComputePatch(computeSelfGromacsPairType);
     // End of JLai
   }
 
   if ( node->simParameters->drudeOn ) {
     mapComputeHomeTuples(computeTholeType);
     mapComputePatch(computeSelfTholeType);
     mapComputeHomeTuples(computeAnisoType);
     mapComputePatch(computeSelfAnisoType);
     mapComputeHomeTuples(computeOneFourNbTholeType);
     mapComputePatch(computeSelfOneFourNbTholeType);
   }
 
   if ( node->simParameters->eFieldOn )
     mapComputePatch(computeEFieldType);
   /* BEGIN gf */
   if ( node->simParameters->mgridforceOn )
     mapComputePatch(computeGridForceType);
   /* END gf */
   if ( node->simParameters->stirOn )
     mapComputePatch(computeStirType);
   if ( node->simParameters->sphericalBCOn )
     mapComputePatch(computeSphericalBCType);
   if ( node->simParameters->cylindricalBCOn )
     mapComputePatch(computeCylindricalBCType);
   if ( node->simParameters->tclBCOn ) {
     mapComputeHomePatches(computeTclBCType);
   }
   if ( node->simParameters->constraintsOn )
     mapComputePatch(computeRestraintsType);
   if ( node->simParameters->consForceOn )
     mapComputePatch(computeConsForceType);
   if ( node->simParameters->consTorqueOn )
     mapComputePatch(computeConsTorqueType);
 
     // store the latest compute map
   SimParameters *simParams = Node::Object()->simParameters;
   if (simParams->storeComputeMap) {
     computeMap->saveComputeMap(simParams->computeMapFilename);
   }
     // override mapping decision
   if (simParams->loadComputeMap) {
     computeMap->loadComputeMap(simParams->computeMapFilename);
     CkPrintf("ComputeMap has been loaded from %s.\n", simParams->computeMapFilename);
   }
 }

◆ messageEnqueueWork()

void WorkDistrib::messageEnqueueWork ( Compute * compute )

static

Definition at line 2866 of file WorkDistrib.C.

References Compute::cid, LocalWorkMsg::compute, computeAnglesType, computeAnisoType, computeBondsType, computeCrosstermsType, computeDihedralsType, computeExclsType, computeGromacsPairType, computeImpropersType, computeLCPOType, computeNonbondedCUDA2Type, computeNonbondedMICType, computeNonbondedPairType, computeNonbondedSelfType, computeOneFourNbTholeType, computePmeCUDAType, computePmeType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfGromacsPairType, computeSelfImpropersType, computeSelfOneFourNbTholeType, computeSelfTholeType, computeTholeType, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), SET_PRIORITY, and Compute::type().

Referenced by cuda_check_pme_forces(), Compute::enqueueWork(), CudaComputeNonbonded::messageEnqueueWork(), ComputePmeMgr::submitReductions(), and ComputePmeMgr::ungridCalc().

                                                      {
   LocalWorkMsg *msg = compute->localWorkMsg;
   int seq = compute->sequence();
   int gbisPhase = compute->getGBISPhase();
 
   if ( seq < 0 ) {
     NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageEnqueueWork");
   } else {
     SET_PRIORITY(msg,seq,compute->priority());
   }
 
   msg->compute = compute; // pointer is valid since send is to local Pe
   int type = compute->type();
   int cid = compute->cid;
 
   CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
   switch ( type ) {
   case computeExclsType:
   case computeSelfExclsType:
     wdProxy[CkMyPe()].enqueueExcls(msg);
     break;
   case computeBondsType:
   case computeSelfBondsType:
     wdProxy[CkMyPe()].enqueueBonds(msg);
     break;
   case computeAnglesType:
   case computeSelfAnglesType:
     wdProxy[CkMyPe()].enqueueAngles(msg);
     break;
   case computeDihedralsType:
   case computeSelfDihedralsType:
     wdProxy[CkMyPe()].enqueueDihedrals(msg);
     break;
   case computeImpropersType:
   case computeSelfImpropersType:
     wdProxy[CkMyPe()].enqueueImpropers(msg);
     break;
   case computeTholeType:
   case computeSelfTholeType:
     wdProxy[CkMyPe()].enqueueThole(msg);
     break;
   case computeAnisoType:
   case computeSelfAnisoType:
     wdProxy[CkMyPe()].enqueueAniso(msg);
     break;
   case computeCrosstermsType:
   case computeSelfCrosstermsType:
     wdProxy[CkMyPe()].enqueueCrossterms(msg);
     break;
   case computeOneFourNbTholeType:
   case computeSelfOneFourNbTholeType:
     wdProxy[CkMyPe()].enqueueOneFourNbThole(msg);
     break;
   // JLai
   case computeGromacsPairType:
   case computeSelfGromacsPairType:
     wdProxy[CkMyPe()].enqueueGromacsPair(msg);
     break;    
   // End of JLai
   case computeLCPOType:
     wdProxy[CkMyPe()].enqueueLCPO(msg);
     break;
   case computeNonbondedSelfType:
     switch ( seq % 2 ) {
     case 0:
       //wdProxy[CkMyPe()].enqueueSelfA(msg);
       switch ( gbisPhase ) {
          case 1:
            wdProxy[CkMyPe()].enqueueSelfA1(msg);
            break;
          case 2:
            wdProxy[CkMyPe()].enqueueSelfA2(msg);
            break;
          case 3:
            wdProxy[CkMyPe()].enqueueSelfA3(msg);
            break;
       }
       break;
     case 1:
       //wdProxy[CkMyPe()].enqueueSelfB(msg);
       switch ( gbisPhase ) {
          case 1:
            wdProxy[CkMyPe()].enqueueSelfB1(msg);
            break;
          case 2:
            wdProxy[CkMyPe()].enqueueSelfB2(msg);
            break;
          case 3:
            wdProxy[CkMyPe()].enqueueSelfB3(msg);
            break;
       }
       break;
     default:
       NAMD_bug("WorkDistrib::messageEnqueueSelf case statement error!");
     }
     break;
   case computeNonbondedPairType:
     switch ( seq % 2 ) {
     case 0:
       //wdProxy[CkMyPe()].enqueueWorkA(msg);
       switch ( gbisPhase ) {
          case 1:
            wdProxy[CkMyPe()].enqueueWorkA1(msg);
            break;
          case 2:
            wdProxy[CkMyPe()].enqueueWorkA2(msg);
            break;
          case 3:
            wdProxy[CkMyPe()].enqueueWorkA3(msg);
            break;
       }
       break;
     case 1:
       //wdProxy[CkMyPe()].enqueueWorkB(msg);
       switch ( gbisPhase ) {
          case 1:
            wdProxy[CkMyPe()].enqueueWorkB1(msg);
            break;
          case 2:
            wdProxy[CkMyPe()].enqueueWorkB2(msg);
            break;
          case 3:
            wdProxy[CkMyPe()].enqueueWorkB3(msg);
            break;
       }
       break;
     case 2:
       wdProxy[CkMyPe()].enqueueWorkC(msg);
       break;
     default:
       NAMD_bug("WorkDistrib::messageEnqueueWork case statement error!");
     }
     break;
 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
   case computeNonbondedCUDA2Type:
 //     CkPrintf("WorkDistrib[%d]::CUDA seq=%d phase=%d\n", CkMyPe(), seq, gbisPhase);
     //wdProxy[CkMyPe()].enqueueCUDA(msg);
     switch ( gbisPhase ) {
        case 1:
          wdProxy[CkMyPe()].enqueueCUDA(msg);
          break;
        case 2:
          wdProxy[CkMyPe()].enqueueCUDAP2(msg);
          break;
        case 3:
          wdProxy[CkMyPe()].enqueueCUDAP3(msg);
          break;
     }
 #else
     msg->compute->doWork();  MACHINE_PROGRESS
 #endif
     break;
   case computeNonbondedMICType:
 #ifdef NAMD_MIC
     wdProxy[CkMyPe()].enqueueMIC(msg);
 #endif
     break;
   case computePmeType:
     // CkPrintf("PME %d %d %x\n", CkMyPe(), seq, compute->priority());
     wdProxy[CkMyPe()].enqueuePme(msg);
     break;
 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
   case computePmeCUDAType:
     wdProxy[CkMyPe()].enqueuePme(msg);
     break;
 #endif
   default:
     wdProxy[CkMyPe()].enqueueWork(msg);
   }
 }

◆ messageFinishCUDA()

void WorkDistrib::messageFinishCUDA ( Compute * compute )

static

Definition at line 3038 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), and SET_PRIORITY.

                                                     {
   LocalWorkMsg *msg = compute->localWorkMsg;
   int seq = compute->sequence();
   int gbisPhase = compute->getGBISPhase();
 
   if ( seq < 0 ) {
     NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageEnqueueWork");
   } else {
     SET_PRIORITY(msg,seq,compute->priority());
   }
 
   msg->compute = compute; // pointer is valid since send is to local Pe
   CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
 
 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
     //wdProxy[CkMyPe()].finishCUDA(msg);
     switch ( gbisPhase ) {
        case 1:
          wdProxy[CkMyPe()].finishCUDA(msg);
          break;
        case 2:
          wdProxy[CkMyPe()].finishCUDAP2(msg);
          break;
        case 3:
          wdProxy[CkMyPe()].finishCUDAP3(msg);
          break;
     }
 #else
     msg->compute->doWork();  MACHINE_PROGRESS
 #endif
 }

◆ messageFinishMIC()

void WorkDistrib::messageFinishMIC ( Compute * compute )

static

Definition at line 3071 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), and SET_PRIORITY.

                                                    {
   LocalWorkMsg *msg = compute->localWorkMsg;
   int seq = compute->sequence();
   int gbisPhase = compute->getGBISPhase();
 
   if ( seq < 0 ) {
     NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageFinishMIC");
   } else {
     SET_PRIORITY(msg,seq,compute->priority());
   }
 
   msg->compute = compute; // pointer is valid since send is to local Pe
   CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
 
 #ifdef NAMD_MIC
     wdProxy[CkMyPe()].finishMIC(msg);
 #else
     msg->compute->doWork();  MACHINE_PROGRESS
 #endif
 }

◆ patchMapInit()

void WorkDistrib::patchMapInit ( void )

Definition at line 1238 of file WorkDistrib.C.

References Lattice::a_p(), Lattice::a_r(), Lattice::b_p(), Lattice::b_r(), Lattice::c_p(), Lattice::c_r(), SimParameters::CUDASOAintegrateMode, DebugM, endi(), PDB::find_extremes(), SimParameters::FMAOn, SimParameters::FMMOn, PDB::get_extremes(), iINFO(), iout, isOutputProcessor(), iWARN(), SimParameters::lattice, SimParameters::LCPOOn, PatchMap::makePatches(), SimParameters::maxPatches, SimParameters::minAtomsPerPatch, Node::molecule, SimParameters::MSMOn, NAMD_die(), SimParameters::noPatchesOnOne, SimParameters::noPatchesOnOutputPEs, SimParameters::noPatchesOnZero, PDB::num_atoms(), Molecule::numAtoms, SimParameters::numoutputprocs, PatchMap::Object(), Node::Object(), Lattice::origin(), SimParameters::patchDimension, Node::pdb, SimParameters::replicaUniformPatchGrids, Node::simParameters, SimParameters::simulatedPEs, SimParameters::simulateInitialMapping, PatchMap::sizeGrid(), SimParameters::staticAtomAssignment, SimParameters::twoAwayX, SimParameters::twoAwayY, SimParameters::twoAwayZ, Vector::x, Vector::y, and Vector::z.

Referenced by Node::startup().

 {
   PatchMap *patchMap = PatchMap::Object();
   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
   Node *node = nd.ckLocalBranch();
   SimParameters *params = node->simParameters;
   Lattice lattice = params->lattice;
 
   BigReal patchSize = params->patchDimension;
 
 #ifndef MEM_OPT_VERSION
   const int totalAtoms = node->pdb->num_atoms();
 #else
   const int totalAtoms = node->molecule->numAtoms;
 #endif
 
   ScaledPosition xmin, xmax;
 
   double maxNumPatches = 1.e9;  // need to adjust fractional values
   if ( params->minAtomsPerPatch > 0 )
     maxNumPatches = totalAtoms / params->minAtomsPerPatch;
 
   DebugM(3,"Mapping patches\n");
   if ( lattice.a_p() && lattice.b_p() && lattice.c_p() ) {
     xmin = 0.;  xmax = 0.;
   }
   else if ( params->FMAOn || params->MSMOn || params->FMMOn ) {
   // Need to use full box for FMA to match NAMD 1.X results.
 #if 0
     node->pdb->find_extremes(&(xmin.x),&(xmax.x),lattice.a_r());
     node->pdb->find_extremes(&(xmin.y),&(xmax.y),lattice.b_r());
     node->pdb->find_extremes(&(xmin.z),&(xmax.z),lattice.c_r());
 #endif
     node->pdb->find_extremes(lattice);
     node->pdb->get_extremes(xmin, xmax);
 #if 0
     printf("+++ center=%.4f %.4f %.4f\n",
         lattice.origin().x, lattice.origin().y, lattice.origin().z);
     printf("+++ xmin=%.4f  xmax=%.4f\n", xmin.x, xmax.x);
     printf("+++ ymin=%.4f  ymax=%.4f\n", xmin.y, xmax.y);
     printf("+++ zmin=%.4f  zmax=%.4f\n", xmin.z, xmax.z);
 #endif
   // Otherwise, this allows a small number of stray atoms.
   }
   else {
 #if 0
     node->pdb->find_extremes(&(xmin.x),&(xmax.x),lattice.a_r(),0.9);
     node->pdb->find_extremes(&(xmin.y),&(xmax.y),lattice.b_r(),0.9);
     node->pdb->find_extremes(&(xmin.z),&(xmax.z),lattice.c_r(),0.9);
 #endif
     node->pdb->find_extremes(lattice, 1.0);
     node->pdb->get_extremes(xmin, xmax);
     iout << iINFO << "ORIGINAL ATOMS MINMAX IS " << xmin << "  " << xmax << "\n" << endi;
     double frac = ( (double)totalAtoms - 10000. ) / (double)totalAtoms;
     if ( frac < 0.9 ) { frac = 0.9; }
     node->pdb->find_extremes(lattice, frac);
     node->pdb->get_extremes(xmin, xmax);
     iout << iINFO << "ADJUSTED ATOMS MINMAX IS " << xmin << "  " << xmax << "\n" << endi;
   }
 
 #if 0
   BigReal origin_shift;
   origin_shift = lattice.a_r() * lattice.origin();
   xmin.x -= origin_shift;
   xmax.x -= origin_shift;
   origin_shift = lattice.b_r() * lattice.origin();
   xmin.y -= origin_shift;
   xmax.y -= origin_shift;
   origin_shift = lattice.c_r() * lattice.origin();
   xmin.z -= origin_shift;
   xmax.z -= origin_shift;
 #endif
 
   // SimParameters default is -1 for unset
   int twoAwayX = params->twoAwayX;
   int twoAwayY = params->twoAwayY;
   int twoAwayZ = params->twoAwayZ;
 
   // SASA implementation is not compatible with twoAway patches
   if (params->LCPOOn && patchSize < 32.4) {
     if ( twoAwayX > 0 || twoAwayY > 0 || twoAwayZ > 0 ) {
       iout << iWARN << "Ignoring twoAway[XYZ] due to LCPO SASA implementation.\n" << endi;
     }
     twoAwayX = twoAwayY = twoAwayZ = 0;
   }
 
   // if you think you know what you're doing go right ahead
   if ( twoAwayX > 0 ) maxNumPatches = 1.e9;
   if ( twoAwayY > 0 ) maxNumPatches = 1.e9;
   if ( twoAwayZ > 0 ) maxNumPatches = 1.e9;
   if ( params->maxPatches > 0 ) {
       maxNumPatches = params->maxPatches;
       iout << iINFO << "LIMITING NUMBER OF PATCHES TO " <<
                                 maxNumPatches << "\n" << endi;
   }
 
   int numpes = CkNumPes();
   SimParameters *simparam = Node::Object()->simParameters;
   if(simparam->simulateInitialMapping) {
     numpes = simparam->simulatedPEs;
     delete [] patchMap->nPatchesOnNode;
     patchMap->nPatchesOnNode = new int[numpes];
     memset(patchMap->nPatchesOnNode, 0, numpes*sizeof(int));    
   }
 
 #if defined(NAMD_CUDA) || defined(NAMD_HIP) || defined(NAMD_MIC)
   // for CUDA be sure there are more patches than pes
 
   int numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
   if ( numPatches < numpes && twoAwayX < 0 ) {
     twoAwayX = 1;
     numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
   }
   if ( numPatches < numpes && twoAwayY < 0 ) {
     twoAwayY = 1;
     numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
   }
   if ( numPatches < numpes && twoAwayZ < 0 ) {
     twoAwayZ = 1;
     numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
   }
   if ( numPatches < numpes ) {
     #if defined(NAMD_MIC)
     NAMD_die("MIC-enabled NAMD requires at least one patch per thread.");
     #else
     if (simparam->CUDASOAintegrateMode) {
       NAMD_die("GPU-resident NAMD requires at least one patch per thread.");
     }
     #endif
   }
   if ( numPatches % numpes && numPatches <= 1.4 * numpes ) {
     int exactFit = numPatches - numPatches % numpes;
     int newNumPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,exactFit,params->staticAtomAssignment,
         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
     if ( newNumPatches == exactFit ) {
       iout << iINFO << "REDUCING NUMBER OF PATCHES TO IMPROVE LOAD BALANCE\n" << endi;
       maxNumPatches = exactFit;
     }
   }
 
   patchMap->makePatches(xmin,xmax,lattice,patchSize,maxNumPatches,
         params->staticAtomAssignment, params->replicaUniformPatchGrids, params->LCPOOn,
         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
 
 #else
 
   int availPes = numpes;
   if ( params->noPatchesOnZero && numpes > 1 ) {
       availPes -= 1;
       if(params->noPatchesOnOne && numpes > 2)
         availPes -= 1;
   }
 #ifdef MEM_OPT_VERSION
   if(params->noPatchesOnOutputPEs && numpes - params->numoutputprocs >2)
     {
       availPes -= params->numoutputprocs;
       if ( params->noPatchesOnZero && numpes > 1 && isOutputProcessor(0)){
         availPes++;
       }
       if ( params->noPatchesOnOne && numpes > 2 && isOutputProcessor(1)){
         availPes++;
       }
     }
 #endif
 
   int numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
   if ( ( numPatches > (0.3*availPes) || numPatches > maxNumPatches
        ) && twoAwayZ < 0 ) {
     twoAwayZ = 0;
     numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
   }
   if ( ( numPatches > (0.6*availPes) || numPatches > maxNumPatches
        ) && twoAwayY < 0 ) {
     twoAwayY = 0;
     numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
   }
   if ( ( numPatches > availPes || numPatches > maxNumPatches
        ) && twoAwayX < 0 ) {
     twoAwayX = 0;
     numPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
   }
   if ( numPatches > availPes && numPatches <= (1.4*availPes) && availPes <= maxNumPatches ) {
     int newNumPatches = patchMap->sizeGrid(
         xmin,xmax,lattice,patchSize,availPes,params->staticAtomAssignment,
         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
     if ( newNumPatches <= availPes && numPatches <= (1.4*newNumPatches) ) {
       iout << iINFO << "REDUCING NUMBER OF PATCHES TO IMPROVE LOAD BALANCE\n" << endi;
       maxNumPatches = availPes;
     }
   }
 
   patchMap->makePatches(xmin,xmax,lattice,patchSize,maxNumPatches,
         params->staticAtomAssignment, params->replicaUniformPatchGrids, params->LCPOOn,
         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
 
 #endif
 
 }

◆ peOrderingReady()

void WorkDistrib::peOrderingReady ( )

static

Definition at line 173 of file WorkDistrib.C.

References cuda_initialize(), and mic_initialize().

Referenced by buildNodeAwarePeOrdering().

                                   {
   //CkPrintf("WorkDistrib::peOrderingReady on %d\n", CkMyPe());
 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
   cuda_initialize();
 #endif
 #ifdef NAMD_MIC
   mic_initialize();
 #endif
 }

◆ recvComputeMapChanges()

void WorkDistrib::recvComputeMapChanges ( ComputeMapChangeMsg * msg )

Definition at line 377 of file WorkDistrib.C.

References ALLBUTME, BUFSIZE, COMPUTEMAPTAG, MOStream::end(), MIStream::get(), NAMD_bug(), ComputeMap::newNode(), ComputeMap::newNumPartitions(), ComputeMap::numComputes(), ComputeMap::Object(), MOStream::put(), ComputeMap::setNewNode(), and ComputeMap::setNewNumPartitions().

                                                                 {
 
   delete msg;
 
   ComputeMap *computeMap = ComputeMap::Object();
 
   int i;
   int nc = computeMap->numComputes();
   
   if ( ! CkMyPe() ) { // send
     // CkPrintf("At %f on %d WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
     MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, COMPUTEMAPTAG, BUFSIZE);
     msg->put(nc);
     for (i=0; i<nc; i++) {
       int data = computeMap->newNode(i);
       msg->put(data);
     }
     msg->put(nc);
     for (i=0; i<nc; i++) {
       char data = computeMap->newNumPartitions(i);
       msg->put(data);
     }
     msg->put(nc);
     msg->end();
     delete msg;
     // CkPrintf("At %f on %d done WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
   } else if ( ! CkMyRank() ) { // receive
     // if ( CkMyNode() == 1 ) CkPrintf("At %f on %d WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
     MIStream *msg = CkpvAccess(comm)->newInputStream(0, COMPUTEMAPTAG);
     msg->get(i);
     if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 1 failed\n");
     for (i=0; i<nc; i++) {
       int data;
       msg->get(data);
       computeMap->setNewNode(i,data);
     }
     msg->get(i);
     if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 2 failed\n");
     for (i=0; i<nc; i++) {
       char data;
       msg->get(data);
       computeMap->setNewNumPartitions(i,data);
     }
     msg->get(i);
     if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 3 failed\n");
     delete msg;
     // if ( CkMyNode() == 1 ) CkPrintf("At %f on %d done WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
   }
 
   CkCallback cb(CkIndex_WorkDistrib::doneSaveComputeMap(NULL), 0, thisgroup);
   contribute(0, NULL, CkReduction::random, cb);
 }

◆ reinitAtoms()

void WorkDistrib::reinitAtoms ( const char * basename = 0 )

Definition at line 1085 of file WorkDistrib.C.

References createAtomLists(), PatchMap::numPatches(), PatchMap::Object(), and PatchMgr::sendAtoms().

                                                   {
 
   PatchMap *patchMap = PatchMap::Object();
   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
   PatchMgr *patchMgr = pm.ckLocalBranch();
 
   int numPatches = patchMap->numPatches();
 
   FullAtomList *atoms = createAtomLists(basename);
 
   for(int i=0; i < numPatches; i++) {
     patchMgr->sendAtoms(i,atoms[i]);
   }
 
   delete [] atoms;
 
 }

◆ saveComputeMapChanges()

void WorkDistrib::saveComputeMapChanges	(	int	ep,
		CkGroupID	chareID
	)

Definition at line 359 of file WorkDistrib.C.

Referenced by ComputeMgr::updateComputes2().

 {
   saveComputeMapReturnEP = ep;
   saveComputeMapReturnChareID = chareID;
 
   ComputeMapChangeMsg *mapMsg = new (0, 0, 0) ComputeMapChangeMsg;
   CProxy_WorkDistrib(thisgroup).recvComputeMapChanges(mapMsg);
 
 /*
     // store the latest compute map
   SimParameters *simParams = Node::Object()->simParameters;
   if (simParams->storeComputeMap) {
     computeMap->saveComputeMap(simParams->computeMapFilename);
     CkPrintf("ComputeMap has been stored in %s.\n", simParams->computeMapFilename);
   }
 */
 }

◆ savePatchMap()

void WorkDistrib::savePatchMap ( PatchMapMsg * msg )

Definition at line 1147 of file WorkDistrib.C.

References SimParameters::isRecvSpanningTreeUnset(), SimParameters::isSendSpanningTreeUnset(), PatchMap::Object(), ProxyMgr::Object(), PatchMapMsg::patchMapData, ProxyMgr::setRecvSpanning(), ProxyMgr::setSendSpanning(), Node::simParameters, split(), and PatchMap::unpack().

 {
   // Use a resend to forward messages before processing.  Otherwise the
   // map distribution is slow on many CPUs.  We need to use a tree
   // rather than a broadcast because some implementations of broadcast
   // generate a copy of the message on the sender for each recipient.
   // This is because MPI doesn't allow re-use of an outstanding buffer.
 
   if ( CkMyRank() ) patchMapArrived = true;
 
   if ( patchMapArrived && CkMyPe() ) {
     PatchMap::Object()->unpack(msg->patchMapData);
 
     //Automatically enable spanning tree
     CProxy_Node nd(CkpvAccess(BOCclass_group).node);
     Node *node = nd.ckLocalBranch();
     SimParameters *params = node->simParameters;
     if( ( PatchMap::Object()->numPatches() <= CkNumPes()/4
 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
         || CkNumPes() > CkNumNodes()
         ) && ( CkNumNodes() > 1
 #endif
       ) && params->isSendSpanningTreeUnset() )
       ProxyMgr::Object()->setSendSpanning();
 
 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
     if ( CkNumPes() > CkNumNodes() && CkNumNodes() > 1
           && params->isRecvSpanningTreeUnset() )
       ProxyMgr::Object()->setRecvSpanning();
 #endif
   }
 
   if ( patchMapArrived ) {
     if ( CkMyRank() + 1 < CkNodeSize(CkMyNode()) ) {
       ((CProxy_WorkDistrib(thisgroup))[CkMyPe()+1]).savePatchMap(msg);
     } else {
       delete msg;
     }
     return;
   }
 
   patchMapArrived = true;
 
   int self = CkMyNode();
   int range_begin = 0;
   int range_end = CkNumNodes();
   while ( self != range_begin ) {
     ++range_begin;
     int split = range_begin + ( range_end - range_begin ) / 2;
     if ( self < split ) { range_end = split; }
     else { range_begin = split; }
   }
   int send_near = self + 1;
   int send_far = send_near + ( range_end - send_near ) / 2;
 
   int pids[3];
   int npid = 0;
   if ( send_far < range_end ) pids[npid++] = CkNodeFirst(send_far);
   if ( send_near < send_far ) pids[npid++] = CkNodeFirst(send_near);
   pids[npid++] = CkMyPe();  // always send the message to ourselves
   CProxy_WorkDistrib(thisgroup).savePatchMap(msg,npid,pids);
 }

◆ send_contributeHostDeviceLDB()

void WorkDistrib::send_contributeHostDeviceLDB	(	int	peSetLen,
		int *	peSet
	)

static

Definition at line 3532 of file WorkDistrib.C.

                                                                         {
   #if defined(NAMD_MIC)
     CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
     wdProxy[0].contributeHostDeviceLDB(peSetLen, peSet);
   #endif
 }

◆ send_initHostDeviceLDB()

void WorkDistrib::send_initHostDeviceLDB ( )

static

Definition at line 3519 of file WorkDistrib.C.

                                          {
   #if defined(NAMD_MIC)
     CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
     wdProxy.initHostDeviceLDB();
   #endif
 }

◆ send_setDeviceLDBParams()

void WorkDistrib::send_setDeviceLDBParams	(	int	dt,
		int	hs,
		int	sp1,
		int	pp1,
		int	pp2
	)

static

Definition at line 3545 of file WorkDistrib.C.

                                                                                    {
   #if defined(NAMD_MIC)
     CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
     wdProxy.setDeviceLDBParams(dt, hs, sp1, pp1, pp2);
   #endif
 }

◆ sendComputeMap()

void WorkDistrib::sendComputeMap ( void )

Definition at line 1211 of file WorkDistrib.C.

References ALLBUTME, BUFSIZE, COMPUTEMAPTAG, MOStream::end(), ComputeMap::initPtrs(), ComputeMap::Object(), ComputeMap::pack(), and ComputeMap::unpack().

Referenced by Node::startup().

 {
   if ( CkMyRank() ) return;
 
   if ( CkNumNodes() == 1 ) {
     computeMapArrived = true;
     ComputeMap::Object()->initPtrs();
     return;
   }
 
   if ( ! CkMyPe() ) { // send
     MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, COMPUTEMAPTAG, BUFSIZE);
     ComputeMap::Object()->pack(msg);
     msg->end();
     delete msg;
   } else if ( ! CkMyRank() ) { // receive
     MIStream *msg = CkpvAccess(comm)->newInputStream(0, COMPUTEMAPTAG);
     ComputeMap::Object()->unpack(msg);
     delete msg;
   }
 
   computeMapArrived = true;
   ComputeMap::Object()->initPtrs();
 }

◆ sendPatchMap()

void WorkDistrib::sendPatchMap ( void )

Definition at line 1111 of file WorkDistrib.C.

References SimParameters::isRecvSpanningTreeUnset(), SimParameters::isSendSpanningTreeUnset(), PatchMap::Object(), ProxyMgr::Object(), PatchMap::pack(), PatchMap::packSize(), PatchMapMsg::patchMapData, ProxyMgr::setRecvSpanning(), ProxyMgr::setSendSpanning(), and Node::simParameters.

Referenced by Node::startup().

 {
   if ( CkNumPes() == 1 ) {
     patchMapArrived = true;
     return;
   }
 
   //Automatically enable spanning tree
   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
   Node *node = nd.ckLocalBranch();
   SimParameters *params = node->simParameters;
   if( ( PatchMap::Object()->numPatches() <= CkNumPes()/4
 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
       || CkNumPes() > CkNumNodes()
       ) && ( CkNumNodes() > 1
 #endif
     ) && params->isSendSpanningTreeUnset() )
     ProxyMgr::Object()->setSendSpanning();
 
 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
   if ( CkNumPes() > CkNumNodes() && CkNumNodes() > 1
         && params->isRecvSpanningTreeUnset() )
     ProxyMgr::Object()->setRecvSpanning();
 #endif
 
   int size = PatchMap::Object()->packSize();
 
   PatchMapMsg *mapMsg = new (size, 0) PatchMapMsg;
 
   PatchMap::Object()->pack(mapMsg->patchMapData, size);
 
   CProxy_WorkDistrib workProxy(thisgroup);
   workProxy[0].savePatchMap(mapMsg);
 }

◆ setDeviceLDBParams()

void WorkDistrib::setDeviceLDBParams	(	int	dt,
		int	hs,
		int	sp1,
		int	pp1,
		int	pp2
	)

Definition at line 3552 of file WorkDistrib.C.

                                                                               {
   #if defined(NAMD_MIC)
     mic_setDeviceLDBParams(dt, hs, sp1, pp1, pp2);
   #endif
 }

◆ setPatchMapArrived()

void WorkDistrib::setPatchMapArrived ( bool s )

inline

Definition at line 108 of file WorkDistrib.h.

Referenced by Node::startup().

108 {patchMapArrived=s;}

◆ sortPmePes()

void WorkDistrib::sortPmePes	(	int *	pmepes,
		int	xdim,
		int	ydim
	)

static

Definition at line 307 of file WorkDistrib.C.

References ResizeArray< Elem >::begin(), PatchMap::center(), PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), and recursive_bisect_coord().

Referenced by ComputePmeMgr::initialize().

                                                             {
   int numpes = CkNumPes();
   ResizeArray<int> count(numpes);
   ResizeArray<ScaledPosition> sumPos(numpes);
   ResizeArray<ScaledPosition> avgPos(numpes);
   for ( int i=0; i<numpes; ++i ) {
     count[i] = 0;
     sumPos[i] = 0;
     avgPos[i] = 0;
   }
   PatchMap *patchMap = PatchMap::Object();
   for ( int i=0, npatches=patchMap->numPatches(); i<npatches; ++i ) {
     int pe = patchMap->node(i);
     count[pe] += 1;
     sumPos[pe] += patchMap->center(i);
   }
   const int npmepes = xdim*ydim;
   ResizeArray<int> sortpes(npmepes);
   for ( int i=0; i<npmepes; ++i ) {
     int pe = sortpes[i] = pmepes[i];
     int cnt = count[pe];
     ScaledPosition sum = sumPos[pe];
     if ( cnt == 0 ) {
       // average over node
       int node = CkNodeOf(pe);
       int nsize = CkNodeSize(node);
       int pe2 = CkNodeFirst(node);
       for ( int j=0; j<nsize; ++j, ++pe2 )  {
         cnt += count[pe2];
         sum += sumPos[pe2];
       }
     }
     if ( cnt == 0 ) {
       // average over physical node
       int node = CmiPhysicalNodeID(pe);
       int nsize, *nlist;
       CmiGetPesOnPhysicalNode(node, &nlist, &nsize);
       for ( int j=0; j<nsize; ++j )  {
         int pe2 = nlist[j];
         cnt += count[pe2];
         sum += sumPos[pe2];
       }
     }
     if ( cnt ) {
       avgPos[pe] = sum / cnt;
     }
   }
   recursive_bisect_coord(0, xdim, 0, ydim, sortpes.begin(), avgPos.begin(), pmepes, ydim);
 }

Member Data Documentation

◆ peCompactOrdering

int * WorkDistrib::peCompactOrdering

static

Definition at line 118 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering().

◆ peCompactOrderingIndex

int * WorkDistrib::peCompactOrderingIndex

static

Definition at line 119 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering(), ParallelIOMgr::initialize(), WorkDistrib::pe_sortop_compact::operator()(), and TopoManagerWrapper::pe_sortop_topo::operator()().

◆ peDiffuseOrdering

int * WorkDistrib::peDiffuseOrdering

static

Definition at line 116 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering(), generatePmePeList2(), ParallelIOMgr::initialize(), and ComputePmeMgr::initialize().

◆ peDiffuseOrderingIndex

int * WorkDistrib::peDiffuseOrderingIndex

static

Definition at line 117 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering(), and WorkDistrib::pe_sortop_diffuse::operator()().

◆ peOrderingInit

int WorkDistrib::peOrderingInit

static

Definition at line 115 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering().

The documentation for this class was generated from the following files:

Classes

Public Member Functions

Static Public Member Functions

Static Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ WorkDistrib()

◆ ~WorkDistrib()

Member Function Documentation

◆ assignNodeToPatch()

◆ buildNodeAwarePeOrdering()

◆ contributeHostDeviceLDB()

◆ createAtomLists()

◆ createHomePatches()

◆ distributeHomePatches()

◆ doneSaveComputeMap()

◆ enqueueAngles()

◆ enqueueAniso()

◆ enqueueBonds()

◆ enqueueCrossterms()

◆ enqueueCUDA()

◆ enqueueCUDAP2()

◆ enqueueCUDAP3()

◆ enqueueDihedrals()

◆ enqueueExcls()

◆ enqueueGromacsPair()

◆ enqueueImpropers()

◆ enqueueLCPO()

◆ enqueueMIC()

◆ enqueueOneFourNbThole()

◆ enqueuePme()

◆ enqueueSelfA1()

◆ enqueueSelfA2()

◆ enqueueSelfA3()

◆ enqueueSelfB1()

◆ enqueueSelfB2()

◆ enqueueSelfB3()

◆ enqueueThole()

◆ enqueueWork()

◆ enqueueWorkA1()

◆ enqueueWorkA2()

◆ enqueueWorkA3()

◆ enqueueWorkB1()

◆ enqueueWorkB2()

◆ enqueueWorkB3()

◆ enqueueWorkC()

◆ finishCUDA()

◆ finishCUDAP2()

◆ finishCUDAP3()

◆ finishCUDAPatch()

◆ finishMIC()

◆ initHostDeviceLDB()

◆ mapComputes()

◆ messageEnqueueWork()

◆ messageFinishCUDA()

◆ messageFinishMIC()

◆ patchMapInit()

◆ peOrderingReady()

◆ recvComputeMapChanges()

◆ reinitAtoms()

◆ saveComputeMapChanges()

◆ savePatchMap()

◆ send_contributeHostDeviceLDB()

◆ send_initHostDeviceLDB()

◆ send_setDeviceLDBParams()

◆ sendComputeMap()

◆ sendPatchMap()

◆ setDeviceLDBParams()

◆ setPatchMapArrived()

◆ sortPmePes()

Member Data Documentation

◆ peCompactOrdering

◆ peCompactOrderingIndex

◆ peDiffuseOrdering

◆ peDiffuseOrderingIndex

◆ peOrderingInit