#include <LdbCoordinator.h>

Inheritance diagram for LdbCoordinator:

Public Member Functions
	LdbCoordinator ()

	~LdbCoordinator (void)

void	initialize (PatchMap pmap, ComputeMap cmap, int reinit=0)

void	createLoadBalancer ()

void	patchLoad (PatchID id, int nAtoms, int timestep)

void	startWork (const LDObjHandle &handle)

void	pauseWork (const LDObjHandle &handle)

void	skipWork (const LDObjHandle &handle)

void	endWork (const LDObjHandle &handle)

void	rebalance (Sequencer *seq, PatchID id)

void	rebalance (Controller *seq)

void	nodeDone (CkReductionMsg *)

void	updateComputesReady ()

void	barrier (void)

void	resume (void)

void	resumeReady (CkQdMsg *msg)

void	resume2 (void)

int	getNumStepsToRun (void)

void	AtSyncBarrierReached (void)

void	Migrate (LDObjHandle handle, int dest)

void	RecvMigrate (LdbMigrateMsg *)

void	ExpectMigrate (LdbMigrateMsg *)

void	ResumeFromSync (void)

void	ExecuteMigrations (void)

void	awakenSequencers (void)

int	requiredProxies (PatchID id, int[])

void	printRequiredProxies (PatchID id, FILE *fp)

void	printLocalLdbReport (void)

void	sendCollectLoads (CollectLoadsMsg *)

void	collectLoads (CollectLoadsMsg *)

Static Public Member Functions
static LdbCoordinator *	Object ()

static void	staticMigrateFn (LDObjHandle handle, int dest)

static void	staticStatsFn (LDOMHandle h, int state)

static void	staticQueryEstLoadFn (LDOMHandle h)

static void	staticReceiveAtSync (void *data)

static void	staticResumeFromSync (void *data)

Public Attributes
int	stepsPerLdbCycle

int	nLocalComputes

int	nLocalPatches

int	nPatchesReported

int	nPatchesExpected

int	nComputesReported

int	nComputesExpected

int	controllerReported

int	controllerExpected

int	nStatsMessagesReceived

int	nStatsMessagesExpected

ComputeMap *	computeMap

PatchMap *	patchMap

int *	patchNAtoms

int	nPatches

Controller *	controllerThread

Sequencer **	sequencerThreads

int	ldbCycleNum

int	numStepsToRun

int	firstLdbStep

int	totalStepsDone

int	takingLdbData

FILE *	ldbStatsFP

computeInfo *	computeArray

patchInfo *	patchArray

processorInfo *	processorArray

LdbInfra *	theLbdb

LDBarrierClient	ldBarrierHandle

LDOMid	myOMid

LDOMHandle	myHandle

LdbMigrateMsg *	migrateMsgs

int	numComputes

int	nRegisteredObjs

int	reg_all_objs

LDObjHandle *	patchHandles

Detailed Description

Definition at line 88 of file LdbCoordinator.h.

Constructor & Destructor Documentation

LdbCoordinator::LdbCoordinator ( )

Definition at line 118 of file LdbCoordinator.C.

References AtSyncBarrierReached(), computeArray, ldBarrierHandle, ldbCycleNum, ldbStatsFP, migrateMsgs, myHandle, myOMid, NAMD_bug(), nLocalComputes, nLocalPatches, numComputes, patchArray, patchNAtoms, processorArray, reg_all_objs, ResumeFromSync(), sequencerThreads, staticMigrateFn(), staticQueryEstLoadFn(), staticReceiveAtSync(), staticResumeFromSync(), staticStatsFn(), takingLdbData, theLbdb, and totalStepsDone.

 {
   if (CkpvAccess(LdbCoordinator_instance) == NULL) {
     CkpvAccess(LdbCoordinator_instance) = this;
   } else {
     NAMD_bug("LdbCoordinator instanced twice on same node!");
   }
   
   collPes = 0;
   ldbCycleNum = 1;
   takingLdbData = 1;
   totalStepsDone = 0;
   nLocalComputes = nLocalPatches = 0;
   patchNAtoms = (int *) NULL;
   sequencerThreads = (Sequencer **) NULL;
   ldbStatsFP = NULL;
   computeArray = NULL;
   patchArray = NULL;
   processorArray = NULL;
 
   // Register self as an object manager for new charm++ balancer framework
   theLbdb = LdbInfra::Object();
 
   // Set the load balancing period (in seconds).  Without this the
   // load balancing framework will hang until 1 second has passed
   // since the last load balancing, causing hiccups in very fast runs.
   // Unfortunately, the clock is already set for the first load
   // balancing, but only +LBPeriod 1.0e-5 can fix that in older charm.
   // For newer versions this is handled in initproc above.
 
 #ifndef LB_MANAGER_VERSION
   theLbdb->SetLBPeriod(1.0e-5);
 #endif
 
   myOMid.id.idx = 1;
   LDCallbacks cb = { (LDMigrateFn)staticMigrateFn,
                      (LDStatsFn)staticStatsFn,
                      (LDQueryEstLoadFn)staticQueryEstLoadFn
                    };
   myHandle = theLbdb->RegisterOM(myOMid,nullptr,cb);
 
 #ifdef LB_MANAGER_VERSION
   // Add myself as a local barrier receiver, so I know when I might
   // be registering objects.
   theLbdb->AddLocalBarrierReceiver(this, &LdbCoordinator::AtSyncBarrierReached);
 
   // Also, add a local barrier client, to trigger load balancing
   ldBarrierHandle = theLbdb->
     AddLocalBarrierClient(this, &LdbCoordinator::ResumeFromSync);
 #else
   // Add myself as a local barrier receiver, so I know when I might
   // be registering objects.
   theLbdb->AddLocalBarrierReceiver((LDBarrierFn)staticReceiveAtSync,
                                    (void*)this);;
 
   // Also, add a local barrier client, to trigger load balancing
   ldBarrierHandle = theLbdb->
     AddLocalBarrierClient((LDResumeFn)staticResumeFromSync,
                           (void*)this);
 #endif
   migrateMsgs = 0; // linked list
   numComputes = 0;
   reg_all_objs = 1;
 }

LdbCoordinator::~LdbCoordinator ( void )

Definition at line 183 of file LdbCoordinator.C.

References computeArray, ldbStatsFP, patchArray, patchNAtoms, processorArray, and sequencerThreads.

 {
   delete [] patchNAtoms;
   delete [] sequencerThreads;
   if (CkMyPe() == 0)
   {
     delete [] computeArray;
     delete [] patchArray;
     delete [] processorArray;
   }
   if (ldbStatsFP)
     fclose(ldbStatsFP);
 
 }

Member Function Documentation

void LdbCoordinator::AtSyncBarrierReached ( void )

Definition at line 101 of file LdbCoordinator.C.

References myHandle, and theLbdb.

Referenced by LdbCoordinator().

 {
   theLbdb->RegisteringObjects(myHandle);
 }

void LdbCoordinator::awakenSequencers ( void )

Definition at line 746 of file LdbCoordinator.C.

References Sequencer::awaken(), Controller::awaken(), controllerThread, PatchMap::numPatches(), patchMap, and sequencerThreads.

Referenced by resume2().

 {
   if (controllerThread)
   {
     controllerThread->awaken();
     controllerThread = NULL;
   }
   for(int i=0; i < patchMap->numPatches(); i++)
   {
     if (sequencerThreads[i])
     {
       sequencerThreads[i]->awaken();
     }
     sequencerThreads[i]= NULL;
   }
 }

void LdbCoordinator::barrier ( void )

Definition at line 647 of file LdbCoordinator.C.

References controllerExpected, controllerReported, ldBarrierHandle, NAMD_bug(), nComputesExpected, nComputesReported, nPatchesExpected, nPatchesReported, and theLbdb.

 {
   if ( (nPatchesReported != nPatchesExpected) 
        || (nComputesReported != nComputesExpected)
        || (controllerReported != controllerExpected) )
   {
     NAMD_bug("Load balancer received wrong number of events.\n");
   }
   theLbdb->AtLocalBarrier(ldBarrierHandle);
 }

void LdbCoordinator::collectLoads ( CollectLoadsMsg * msg )

Definition at line 839 of file LdbCoordinator.C.

References COLL_AVG, COLL_MAX, COLL_SUM, endi(), CollectLoadsMsg::finalTime, CollectLoadsMsg::firstPe, CollectLoadsMsg::initTime, iout, CollectLoadsMsg::lastPe, and CollectLoadsMsg::strategyName.

                                                       {
   // CkPrintf("LdbCoordinator::collectLoads recv %d-%d\n", msg->firstPe, msg->lastPe);
   if ( collPes == 0 ) {
     reverted = 0;
     initTotalProxies = 0;
     finalTotalProxies = 0;
     initMaxPeProxies = 0;
     finalMaxPeProxies = 0;
     initMaxPatchProxies = 0;
     finalMaxPatchProxies = 0;
     initTime = 0;
     finalTime = 0;
     initMemory = 0;
     finalMemory = 0;
     initAvgPeLoad = 0;
     finalAvgPeLoad = 0;
     initMaxPeLoad = 0;
     finalMaxPeLoad = 0;
   }
   int numPes = msg->lastPe - msg->firstPe + 1;
   collPes += numPes;
 #define COLL_MAX(F) if ( msg->F > F ) F = msg->F;
 #define COLL_AVG(F) F += msg->F * (double) numPes / (double) CkNumPes();
 #define COLL_SUM(F) F += msg->F;
   COLL_SUM(reverted)
   COLL_SUM(initTotalProxies)
   COLL_SUM(finalTotalProxies)
   COLL_MAX(initMaxPeProxies)
   COLL_MAX(finalMaxPeProxies)
   COLL_MAX(initMaxPatchProxies)
   COLL_MAX(finalMaxPatchProxies)
   if ( (msg->finalTime - msg->initTime) > (finalTime - initTime) ) {
     initTime = msg->initTime;
     finalTime = msg->finalTime;
   }
   COLL_MAX(initMemory)
   COLL_MAX(finalMemory)
   COLL_AVG(initAvgPeLoad)
   COLL_AVG(finalAvgPeLoad)
   COLL_MAX(initMaxPeLoad)
   COLL_MAX(finalMaxPeLoad)
 
   if ( collPes == CkNumPes() ) {
     collPes = 0;
     iout << "LDB: TIME " << initTime << " LOAD: AVG " << initAvgPeLoad
       << " MAX " << initMaxPeLoad << "  PROXIES: TOTAL " << initTotalProxies << " MAXPE " <<
       initMaxPeProxies << " MAXPATCH " << initMaxPatchProxies << " " << "None"
       << " MEM: " << initMemory << " MB\n";
     if ( reverted ) iout << "LDB: Reverting to original mapping on " << reverted << " balancers\n";
     iout << "LDB: TIME " << finalTime << " LOAD: AVG " << finalAvgPeLoad
       << " MAX " << finalMaxPeLoad << "  PROXIES: TOTAL " << finalTotalProxies << " MAXPE " <<
       finalMaxPeProxies << " MAXPATCH " << finalMaxPatchProxies << " " << msg->strategyName
       << " MEM: " << finalMemory << " MB\n";
     iout << endi;
     fflush(stdout);
   }
 
   delete msg;
 }

void LdbCoordinator::createLoadBalancer ( )

Definition at line 198 of file LdbCoordinator.C.

References CreateNamdCentLB(), CreateNamdHybridLB(), LDBAL_CENTRALIZED, LDBAL_HYBRID, SimParameters::ldBalancer, Node::Object(), Node::simParameters, and simParams.

Referenced by Node::startup().

 {
   const SimParameters *simParams = Node::Object()->simParameters;
 
   // Create hierarchical or centralized load balancers
   // Currently centralized is the default
   if (simParams->ldBalancer == LDBAL_CENTRALIZED) {
     CkPrintf("LDB: Central LB being created...\n");
     CreateNamdCentLB();
   } else if (simParams->ldBalancer == LDBAL_HYBRID) {
     CkPrintf("LDB: Hybrid LB being created...\n");
     CreateNamdHybridLB();
   }
 }

void LdbCoordinator::endWork ( const LDObjHandle & handle )

inline

Definition at line 110 of file LdbCoordinator.h.

References nComputesReported, and theLbdb.

Referenced by ComputePatch::doWork(), ComputePatchPair::doWork(), ComputeLCPO::doWork(), and ComputeHomeTuples< TholeElem, Thole, TholeValue >::doWork().

                                           {  // both
     theLbdb->ObjectStop(handle);
     nComputesReported++;
   }

void LdbCoordinator::ExecuteMigrations ( void )

Definition at line 670 of file LdbCoordinator.C.

References computeMgr, ComputeMgr::updateComputes(), and updateComputesReady().

Referenced by nodeDone().

 {
  // computeMgr->updateComputes() call only on Node(0) i.e. right here
   // This will barrier for all Nodes - (i.e. Computes must be
   // here and with proxies before anyone can start up
 
   CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
   ComputeMgr *computeMgr = cm.ckLocalBranch();
   computeMgr->updateComputes(CkIndex_LdbCoordinator::
                              updateComputesReady(),thisgroup);
 }

void LdbCoordinator::ExpectMigrate ( LdbMigrateMsg * m )

Definition at line 697 of file LdbCoordinator.C.

References LdbMigrateMsg::from, LdbMigrateMsg::handle, migrateMsgs, myHandle, LdbMigrateMsg::next, and theLbdb.

Referenced by Migrate(), and RecvMigrate().

 {
   if ( m->from != CkMyPe() ) {
     m->handle = theLbdb->RegisterObj(myHandle,m->handle.id,0,1);
     theLbdb->Migrated(m->handle);
   }
 
   m->next = migrateMsgs;
   migrateMsgs = m;
 }

int LdbCoordinator::getNumStepsToRun ( void )

inline

Definition at line 123 of file LdbCoordinator.h.

References numStepsToRun.

Referenced by Sequencer::rebalanceLoad(), and Controller::rebalanceLoad().

123 { return numStepsToRun; }

LdbCoordinator::numStepsToRun

int numStepsToRun

Definition: LdbCoordinator.h:161

void LdbCoordinator::initialize	(	PatchMap *	pmap,
		ComputeMap *	cmap,
		int	reinit = `0`
	)

Definition at line 213 of file LdbCoordinator.C.

References BONDED_TYPE, SimParameters::bondedCUDA, ComputeMap::compute(), computeAnglesType, computeAnisoType, computeArray, computeBondsType, computeCrosstermsType, computeDihedralsType, computeExclsType, computeGromacsPairType, computeImpropersType, computeLCPOType, computeMap, computeNonbondedPairType, computeNonbondedSelfType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfGromacsPairType, computeSelfImpropersType, computeSelfTholeType, computeTholeType, controllerExpected, controllerReported, controllerThread, SimParameters::firstLdbStep, firstLdbStep, LdbMigrateMsg::handle, PatchMap::homePatch(), SimParameters::lastLdbStep, ldbCycleNum, LdbIdField(), SimParameters::ldbPeriod, Compute::ldObjHandle, HomePatch::ldObjHandle, migrateMsgs, SimParameters::multigratorOn, SimParameters::multigratorPressureFreq, myHandle, Node::myid(), NAMD_bug(), NAMD_die(), nComputesExpected, nComputesReported, LdbMigrateMsg::next, nLocalComputes, nLocalPatches, ComputeMap::node(), PatchMap::node(), NONBONDED_OR_SELF_TYPE, nPatches, nPatchesExpected, nPatchesReported, nStatsMessagesExpected, nStatsMessagesReceived, ComputeMap::numComputes(), numComputes, PatchMap::numHomePatches(), Node::numNodes(), PatchMap::numPatches(), ComputeMap::numPids(), numStepsToRun, Node::Object(), PATCH_TYPE, patchArray, patchHandles, patchMap, patchNAtoms, ComputeMap::pid(), processorArray, reg_all_objs, sequencerThreads, Node::simParameters, simParams, SimParameters::stepsPerCycle, stepsPerLdbCycle, takingLdbData, theLbdb, totalStepsDone, and ComputeMap::type().

Referenced by resume(), and Node::startup().

 {
   const SimParameters *simParams = Node::Object()->simParameters;
 
   //  DebugM(10,"stepsPerLdbCycle initialized\n");
   stepsPerLdbCycle = simParams->ldbPeriod;
   firstLdbStep = simParams->firstLdbStep;
   int lastLdbStep = simParams->lastLdbStep;
   int stepsPerCycle = simParams->stepsPerCycle;
 
   computeMap = cMap;
   patchMap = pMap;
 
   // Set the number of received messages correctly for node 0
 
   nStatsMessagesExpected = Node::Object()->numNodes();
   nStatsMessagesReceived = 0;
 
   if (patchNAtoms) 
     delete [] patchNAtoms;  // Depends on delete NULL to do nothing
   nPatches = patchMap->numPatches();
   patchNAtoms = new int[nPatches];
 
   typedef Sequencer *seqPtr;
 
   if ( ! reinit ) {
     delete [] sequencerThreads;  // Depends on delete NULL to do nothing
     sequencerThreads = new seqPtr[nPatches];
   }
 
   nLocalPatches=0;
 
   int i;
   for(i=0;i<nPatches;i++)
   {
     if (patchMap->node(i) == Node::Object()->myid())
     {
       nLocalPatches++;
       patchNAtoms[i]=0;
     } else {
       patchNAtoms[i]=-1;
     }
     if ( ! reinit ) sequencerThreads[i]=NULL;
   }
   if ( ! reinit ) controllerThread = NULL;
   if (nLocalPatches != patchMap->numHomePatches())
     NAMD_die("Disaggreement in patchMap data.\n");
  
   const int oldNumComputes = numComputes;
   nLocalComputes = 0;
   numComputes = computeMap->numComputes();
 
   for(i=0;i<numComputes;i++)  {
     if ( (computeMap->node(i) == Node::Object()->myid())
          && ( 0
               #if (defined(NAMD_CUDA) || defined(NAMD_HIP) || defined(NAMD_MIC))
                 #if defined(NAMD_MIC)
                   || ((computeMap->type(i) == computeNonbondedSelfType) && (computeMap->directToDevice(i) == 0))
                   || ((computeMap->type(i) == computeNonbondedPairType) && (computeMap->directToDevice(i) == 0))
                 #endif
               #else
               || (computeMap->type(i) == computeNonbondedSelfType)
               || (computeMap->type(i) == computeNonbondedPairType)
 #endif
 #if (defined(NAMD_CUDA) || defined(NAMD_HIP)) && defined(BONDED_CUDA)
         || (computeMap->type(i) == computeSelfBondsType && !(simParams->bondedCUDA & 1))
         || (computeMap->type(i) == computeBondsType && !(simParams->bondedCUDA & 1))
         || (computeMap->type(i) == computeSelfAnglesType && !(simParams->bondedCUDA & 2))
         || (computeMap->type(i) == computeAnglesType && !(simParams->bondedCUDA & 2))
         || (computeMap->type(i) == computeSelfDihedralsType && !(simParams->bondedCUDA & 4))
         || (computeMap->type(i) == computeDihedralsType && !(simParams->bondedCUDA & 4))
         || (computeMap->type(i) == computeSelfImpropersType && !(simParams->bondedCUDA & 8))
         || (computeMap->type(i) == computeImpropersType && !(simParams->bondedCUDA & 8))
         || (computeMap->type(i) == computeSelfExclsType && !(simParams->bondedCUDA & 16))
         || (computeMap->type(i) == computeExclsType && !(simParams->bondedCUDA & 16))
         || (computeMap->type(i) == computeSelfCrosstermsType && !(simParams->bondedCUDA & 32))
         || (computeMap->type(i) == computeCrosstermsType && !(simParams->bondedCUDA & 32))
 #else
         || (computeMap->type(i) == computeSelfBondsType)
         || (computeMap->type(i) == computeBondsType)
         || (computeMap->type(i) == computeSelfAnglesType)
         || (computeMap->type(i) == computeAnglesType)
         || (computeMap->type(i) == computeSelfDihedralsType)
         || (computeMap->type(i) == computeDihedralsType)
         || (computeMap->type(i) == computeSelfImpropersType)
         || (computeMap->type(i) == computeImpropersType)
         || (computeMap->type(i) == computeSelfExclsType)
         || (computeMap->type(i) == computeExclsType)
         || (computeMap->type(i) == computeSelfCrosstermsType)
         || (computeMap->type(i) == computeCrosstermsType)
 #endif
               || (computeMap->type(i) == computeLCPOType)
               || (computeMap->type(i) == computeSelfTholeType)
               || (computeMap->type(i) == computeSelfAnisoType)
 
                  || (computeMap->type(i) == computeTholeType)
                  || (computeMap->type(i) == computeAnisoType)
               // JLai
                  || (computeMap->type(i) == computeGromacsPairType)
                  || (computeMap->type(i) == computeSelfGromacsPairType)
         ) ) {
       nLocalComputes++;
     }
   }
   
   // New LB frameworks registration
 
   // Allocate data structure to save incoming migrations.  Processor
   // zero will get all migrations
 
   // If this is the first time through, we need it register patches
   if (ldbCycleNum == reg_all_objs) {
     if ( 1 ) { // ( Node::Object()->simParameters->ldBalancer == LDBAL_CENTRALIZED ) {
       reg_all_objs = 3;
     }
     // Tell the lbdb that I'm registering objects, until I'm done
     // registering them.
     theLbdb->RegisteringObjects(myHandle);
     
    if ( ldbCycleNum == 1 ) {
     patchHandles = new LDObjHandle[nLocalPatches];
     int patch_count=0;
     int i;
     for(i=0;i<nPatches;i++)
       if (patchMap->node(i) == Node::Object()->myid()) {
 
         LdbId elemID;
         LdbIdField(elemID, 0) = i;
         LdbIdField(elemID, 1) = PATCH_TYPE;
 
         if (patch_count >= nLocalPatches) {
     NAMD_bug("LdbCoordinator found too many local patches!");
         }
         HomePatch *p = patchMap->homePatch(i);
         p->ldObjHandle = 
         patchHandles[patch_count] 
           = theLbdb->RegisterObj(myHandle,elemID,0,0);
         patch_count++;
 
       }
    }
   
     if ( numComputes > oldNumComputes ) {
       // Register computes
       for(i=oldNumComputes; i<numComputes; i++)  {
         if ( computeMap->node(i) == Node::Object()->myid())
         {
           if ( 0
                #if (defined(NAMD_CUDA) || defined(NAMD_HIP) || defined(NAMD_MIC))
                  #if defined(NAMD_MIC)
                    || ((computeMap->type(i) == computeNonbondedSelfType) && (computeMap->directToDevice(i) == 0))
                    || ((computeMap->type(i) == computeNonbondedPairType) && (computeMap->directToDevice(i) == 0))
                  #endif
                #else
                   || (computeMap->type(i) == computeNonbondedSelfType)
                   || (computeMap->type(i) == computeNonbondedPairType)
                #endif
 #if (defined(NAMD_CUDA) || defined(NAMD_HIP)) && defined(BONDED_CUDA)
             || (computeMap->type(i) == computeSelfBondsType && !(simParams->bondedCUDA & 1))
             || (computeMap->type(i) == computeSelfAnglesType && !(simParams->bondedCUDA & 2))
             || (computeMap->type(i) == computeSelfDihedralsType && !(simParams->bondedCUDA & 4))
             || (computeMap->type(i) == computeSelfImpropersType && !(simParams->bondedCUDA & 8))
             || (computeMap->type(i) == computeSelfExclsType && !(simParams->bondedCUDA & 16))
             || (computeMap->type(i) == computeSelfCrosstermsType && !(simParams->bondedCUDA & 32))
 #else
             || (computeMap->type(i) == computeSelfBondsType)
             || (computeMap->type(i) == computeSelfAnglesType)
             || (computeMap->type(i) == computeSelfDihedralsType)
             || (computeMap->type(i) == computeSelfImpropersType)
             || (computeMap->type(i) == computeSelfExclsType)
             || (computeMap->type(i) == computeSelfCrosstermsType)
 #endif
                   || (computeMap->type(i) == computeLCPOType)
                   || (computeMap->type(i) == computeSelfTholeType)
                   || (computeMap->type(i) == computeSelfAnisoType)
                // JLai
                   || (computeMap->type(i) == computeSelfGromacsPairType)
                // End of JLai
                 )  {
           // Register the object with the load balancer
           // Store the depended patch IDs in the rest of the element ID
           LdbId elemID;
           LdbIdField(elemID, 0) = i;
 
           if (computeMap->numPids(i) > 0)
             LdbIdField(elemID, 1) =  computeMap->pid(i,0);
           else LdbIdField(elemID, 1) = NONBONDED_OR_SELF_TYPE;
 
           Compute *c = computeMap->compute(i);
           if ( ! c ) NAMD_bug("LdbCoordinator::initialize() null compute pointer");
 
           c->ldObjHandle = theLbdb->RegisterObj(myHandle,elemID,0,1);
           }
           else if ( 
 #if (defined(NAMD_CUDA) || defined(NAMD_HIP)) && defined(BONDED_CUDA)
                     (computeMap->type(i) == computeBondsType && !(simParams->bondedCUDA & 1))
                  || (computeMap->type(i) == computeAnglesType && !(simParams->bondedCUDA & 2))
                  || (computeMap->type(i) == computeDihedralsType && !(simParams->bondedCUDA & 4))
                  || (computeMap->type(i) == computeImpropersType && !(simParams->bondedCUDA & 8))
                  || (computeMap->type(i) == computeExclsType && !(simParams->bondedCUDA & 16))
                  || (computeMap->type(i) == computeCrosstermsType && !(simParams->bondedCUDA & 32))
 #else
                     (computeMap->type(i) == computeBondsType)
                  || (computeMap->type(i) == computeAnglesType)
                  || (computeMap->type(i) == computeDihedralsType)
                  || (computeMap->type(i) == computeImpropersType)
                  || (computeMap->type(i) == computeExclsType)
                  || (computeMap->type(i) == computeCrosstermsType)
 #endif
                  || (computeMap->type(i) == computeTholeType)
                  || (computeMap->type(i) == computeAnisoType)
                  // JLai
                  || (computeMap->type(i) == computeGromacsPairType)
                  // End of JLai
                ) {
           // Register the object with the load balancer
           // Store the depended patch IDs in the rest of the element ID
           LdbId elemID;
           LdbIdField(elemID, 0) = i;
         
           LdbIdField(elemID, 1) = BONDED_TYPE;
 
           Compute *c = computeMap->compute(i);
           if ( ! c ) NAMD_bug("LdbCoordinator::initialize() null compute pointer");
 
           c->ldObjHandle = theLbdb->RegisterObj(myHandle,elemID,0,0);
           }
         }
       }
     }
     theLbdb->DoneRegisteringObjects(myHandle);
   }
 
   // process saved migration messages, if any
   while ( migrateMsgs ) {
     LdbMigrateMsg *m = migrateMsgs;
     migrateMsgs = m->next;
     Compute *c = computeMap->compute(LdbIdField(m->handle.id, 0));
     if ( ! c ) NAMD_bug("LdbCoordinator::initialize() null compute pointer 2");
     c->ldObjHandle = m->handle;
     delete m;
   }
 
   // Fixup to take care of the extra timestep at startup
   // This is pretty ugly here, but it makes the count correct
   
   // iout << "LDB Cycle Num: " << ldbCycleNum << "\n";
 
  if ( 1 ) { // ( simParams->ldBalancer == LDBAL_CENTRALIZED ) {
   if (ldbCycleNum == 1 || ldbCycleNum == 3) {
     numStepsToRun = stepsPerCycle;
     totalStepsDone += numStepsToRun;
     takingLdbData = 0;
     theLbdb->CollectStatsOff();
   } else if (ldbCycleNum == 2 || ldbCycleNum == 4) {
     numStepsToRun = firstLdbStep - stepsPerCycle;
     while ( numStepsToRun <= 0 ) numStepsToRun += stepsPerCycle;
     totalStepsDone += numStepsToRun;
     takingLdbData = 1;
     theLbdb->CollectStatsOn();
   } else if ( (ldbCycleNum <= 6) || !takingLdbData )
   {
     totalStepsDone += firstLdbStep;
     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
       numStepsToRun = -1;
       takingLdbData = 0;
       theLbdb->CollectStatsOff();
     } else {
       numStepsToRun = firstLdbStep;
       takingLdbData = 1;
       theLbdb->CollectStatsOn();
     }
   }
   else 
   {
     totalStepsDone += stepsPerLdbCycle - firstLdbStep;
     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
       numStepsToRun = -1;
       takingLdbData = 0;
       theLbdb->CollectStatsOff();
     } else {
       numStepsToRun = stepsPerLdbCycle - firstLdbStep;
       takingLdbData = 0;
       theLbdb->CollectStatsOff();
     }
   }
  } else {
   if (ldbCycleNum==1)
   {
     totalStepsDone += firstLdbStep;
     numStepsToRun = firstLdbStep;
     takingLdbData = 0;
     theLbdb->CollectStatsOff();
   }
   else if ( (ldbCycleNum <= 4) || !takingLdbData )
   {
     totalStepsDone += firstLdbStep;
     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
       numStepsToRun = -1;
       takingLdbData = 0;
       theLbdb->CollectStatsOff();
     } else {
       numStepsToRun = firstLdbStep;
       takingLdbData = 1;
       theLbdb->CollectStatsOn();
     }
   }
   else 
   {
     totalStepsDone += stepsPerLdbCycle - firstLdbStep;
     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
       numStepsToRun = -1;
       takingLdbData = 0;
       theLbdb->CollectStatsOff();
     } else {
       numStepsToRun = stepsPerLdbCycle - firstLdbStep;
       takingLdbData = 0;
       theLbdb->CollectStatsOff();
     }
   }
  }
 
 /*-----------------------------------------------------------------------------*
  * --------------------------------------------------------------------------- *
  * Comments inserted by Abhinav to clarify relation between ldbCycleNum,       *
  * load balancing step numbers (printed by the step() function) and            *
  * tracing of the steps                                                        *
  * --------------------------------------------------------------------------- *
  * If trace is turned off in the beginning, then tracing is turned on          *
  * at ldbCycleNum = 4 and turned off at ldbCycleNum = 8. ldbCycleNum can       *
  * be adjusted by specifying firstLdbStep and ldbPeriod which are set by       *
  * default to 5*stepspercycle and 200*stepspercycle if not specified.          *
  *                                                                             *
  * If we choose firstLdbStep = 20 and ldbPeriod = 100, we have the             *
  * following timeline (for these particular numbers):                          *
  *                                                                             *
  * Tracing         :  <------ off ------><------------- on -----------><-- off *
  * Ldb Step() No   :              1     2     3        4      5       6      7 *
  * Iteration Steps : 00====20====40====60====80======160====180=====260====280 *
  * ldbCycleNum     :  1     2     3     4     5        6      7       8      9 *
  * Instrumention   :          Inst  Inst  Inst           Inst            Inst  *
  * LDB Strategy    :              TLB  RLB   RLB            RLB            RLB *
  *                                                                             *
  * TLB = TorusLB                                                               *
  * RLB = RefineTorusLB                                                         *
  * Inst = Instrumentation Phase (no real load balancing)                       *
  * --------------------------------------------------------------------------- *
  *-----------------------------------------------------------------------------*
  */
 #if 0 //replaced by traceBarrier at Controller and Sequencer
   if (traceAvailable()) {
     static int specialTracing = 0; // XXX static variables are unsafe for SMP
     if (ldbCycleNum == 1 && traceIsOn() == 0)  specialTracing = 1;
     if (specialTracing) {
       if (ldbCycleNum == 4) traceBegin();
       if (ldbCycleNum == 8) traceEnd();
     }
   }
 #endif
 
   nPatchesReported = 0;
   nPatchesExpected = nLocalPatches;
   nComputesReported = 0;
   nComputesExpected = nLocalComputes * numStepsToRun;
   controllerReported = 0;
   controllerExpected = ! CkMyPe();
 
   if (simParams->multigratorOn) {
     // Add the number of pressure cycles into nComputesExpected:
     // Pressure cycle is done when !(step % simParams->multigratorPressureFreq) = true
     // step = Current step
     int step = totalStepsDone - numStepsToRun;
     int freq = simParams->multigratorPressureFreq;
     // dstep = Number of steps we have to take until next pressure cycle
     int dstep = 0;
     if ((step % freq) != 0) dstep = freq - (step % freq);
     step += dstep;
     if (step < totalStepsDone) {
       int numPressureCycles = 1 + ((totalStepsDone-step-1)/freq);
       if (step==0) numPressureCycles--;
       // if (CkMyPe()==2) fprintf(stderr, "step %d totalStepsDone %d numPressureCycles %d\n",
       //   step, totalStepsDone, numPressureCycles);
       nComputesExpected += 2*nLocalComputes*numPressureCycles;
     }
   }
 
   if (CkMyPe() == 0)
   {
     if (computeArray == NULL)
       computeArray = new computeInfo[numComputes];
     if (patchArray == NULL)
       patchArray = new patchInfo[nPatches];
     if (processorArray == NULL)
       processorArray = new processorInfo[CkNumPes()];
   }
     
   theLbdb->ClearLoads();
 }

void LdbCoordinator::Migrate	(	LDObjHandle	handle,
		int	dest
	)

Definition at line 67 of file LdbCoordinator.C.

References ExpectMigrate(), LdbMigrateMsg::from, LdbMigrateMsg::handle, and LdbMigrateMsg::to.

Referenced by staticMigrateFn(), and ComputeMgr::updateLocalComputes().

 {
   LdbMigrateMsg* msg = new LdbMigrateMsg;
   msg->handle = handle;
   msg->from = CkMyPe();
   msg->to = dest;
   if ( msg->to != CkMyPe() ) {
     CProxy_LdbCoordinator ldbProxy(thisgroup);
     ldbProxy[CkMyPe()].RecvMigrate(msg);
   } else {
     ExpectMigrate(msg);
   }
 }

void LdbCoordinator::nodeDone ( CkReductionMsg * msg )

Definition at line 658 of file LdbCoordinator.C.

References endi(), ExecuteMigrations(), iout, takingLdbData, and updateComputesReady().

 {
   delete msg;
 
   iout << "LDB: ============== END OF LOAD BALANCING =============== " << CmiWallTimer() << "\n" << endi;
   if ( takingLdbData ) {
       ExecuteMigrations();
   } else {
       updateComputesReady();
   }
 }

static LdbCoordinator* LdbCoordinator::Object ( )

inlinestatic

Definition at line 93 of file LdbCoordinator.h.

Referenced by ComputePatch::doWork(), ComputePatchPair::doWork(), ComputeLCPO::doWork(), ComputeHomeTuples< TholeElem, Thole, TholeValue >::doWork(), SimpleBroadcastObject< Tensor >::get(), ComputeNonbondedPair::noWork(), ComputeNonbondedSelf::noWork(), ComputeLCPO::noWork(), Sequencer::rebalanceLoad(), Controller::rebalanceLoad(), Sequencer::Sequencer(), Node::startup(), staticMigrateFn(), HomePatch::submitLoadStats(), Sequencer::suspend(), Sequencer::terminate(), ComputeMgr::updateLocalComputes(), and Rebalancer::~Rebalancer().

                                    { 
     return CkpvAccess(LdbCoordinator_instance); 
   }

void LdbCoordinator::patchLoad	(	PatchID	id,
		int	nAtoms,
		int	timestep
	)

Definition at line 612 of file LdbCoordinator.C.

References DebugM, nPatches, nPatchesReported, and patchNAtoms.

Referenced by HomePatch::submitLoadStats().

 {
   CmiAssert( id >=0 && id < nPatches);
   if (patchNAtoms[id] != -1) {
     patchNAtoms[id] = nAtoms;
     nPatchesReported++;
   } else {
     DebugM(10, "::patchLoad() Unexpected patch reporting in\n");
   }
 }

void LdbCoordinator::pauseWork ( const LDObjHandle & handle )

inline

Definition at line 104 of file LdbCoordinator.h.

References theLbdb.

Referenced by ComputePatch::doWork(), ComputePatchPair::doWork(), SimpleBroadcastObject< Tensor >::get(), Sequencer::suspend(), and Sequencer::terminate().

                                             {  // stop timer only
     theLbdb->ObjectStop(handle);
   }

void LdbCoordinator::printLocalLdbReport ( void )

Definition at line 791 of file LdbCoordinator.C.

References PatchMap::numPatches(), patchMap, and patchNAtoms.

 {
   char outputBuf[255];
   char *curLoc;
 
   CkPrintf("%d:Patch report:\n",CkMyPe());
   
   curLoc = outputBuf;
   int i,j=0;
   for(i=0; i<patchMap->numPatches(); i++)
   {
     if (patchNAtoms[i] != -1)
     {
       curLoc += sprintf(curLoc,"%5d: %5d ",i,patchNAtoms[i]);
       j++;
     } 
     if (((j % 4) == 0) && j)
     {
       curLoc = outputBuf;
       CkPrintf("[%d]%s\n",CkMyPe(),outputBuf);
       j=0;
     }
   }
 
   CkPrintf("%d:Compute report:\n",CkMyPe());
   
   curLoc = outputBuf;
   j=0;
 }

void LdbCoordinator::printRequiredProxies	(	PatchID	id,
		FILE *	fp
	)

Definition at line 821 of file LdbCoordinator.C.

References PatchMap::MaxOneAway, PatchMap::MaxTwoAway, and requiredProxies().

 {
   // Check all two-away neighbors.
   // This is really just one-away neighbors, since 
   // two-away always returns zero: RKB
   int neighborNodes[PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
   const int nProxyNodes = requiredProxies(id,neighborNodes);
 
   fprintf(fp,"%4d ",nProxyNodes);
 
   for(int i=0;i<nProxyNodes;i++)
     fprintf(fp,"%4d ",neighborNodes[i]);
 }

void LdbCoordinator::rebalance	(	Sequencer *	seq,
		PatchID	id
	)

Definition at line 623 of file LdbCoordinator.C.

References LDBAL_NONE, Node::Object(), sequencerThreads, and Sequencer::suspend().

Referenced by Sequencer::rebalanceLoad(), and Controller::rebalanceLoad().

 {
   if (Node::Object()->simParameters->ldBalancer == LDBAL_NONE)
     return;
 
   sequencerThreads[pid] = seq;
   seq->suspend();
 }

void LdbCoordinator::rebalance ( Controller * seq )

Definition at line 632 of file LdbCoordinator.C.

References controllerReported, controllerThread, DebugM, endi(), iout, LDBAL_NONE, and Node::Object().

 {
   if (Node::Object()->simParameters->ldBalancer == LDBAL_NONE)
     return;
 
   iout << "LDB: ============= START OF LOAD BALANCING ============== " << CmiWallTimer() << "\n" << endi;
   DebugM(3, "Controller reached load balance barrier.\n");
   controllerReported = 1;
   controllerThread = c;
 
   CProxy_LdbCoordinator(thisgroup).barrier();
 
   CthSuspend();
 }

void LdbCoordinator::RecvMigrate ( LdbMigrateMsg * m )

Definition at line 682 of file LdbCoordinator.C.

References ExpectMigrate(), LdbMigrateMsg::handle, theLbdb, and LdbMigrateMsg::to.

 {
   // This method receives the migration from the framework,
   // unregisters it, and sends it to the destination PE
 
   if ( m->to != CkMyPe() ) {
     theLbdb->UnregisterObj(m->handle);
 
     CProxy_LdbCoordinator  ldbProxy(thisgroup);
     ldbProxy[m->to].ExpectMigrate(m);
   } else {
     ExpectMigrate(m);
   }
 }

int LdbCoordinator::requiredProxies	(	PatchID	id,
		int	neighborNodes[]
	)

Definition at line 767 of file LdbCoordinator.C.

References PatchMap::basenode(), PatchMap::downstreamNeighbors(), PatchMap::MaxOneAway, PatchMap::node(), and patchMap.

Referenced by printRequiredProxies().

 {
   PatchID neighbors[1 + PatchMap::MaxOneAway];
   neighbors[0] = id;
   int numNeighbors = 1 + patchMap->downstreamNeighbors(id,neighbors+1);
 
   int nProxyNodes = 0;
   int myNode = patchMap->node(id);
   for ( int i = 0; i < numNeighbors; ++i ) {
     const int proxyNode = patchMap->basenode(neighbors[i]);
     if ( proxyNode != myNode ) {
       int j;
       for ( j = 0; j < nProxyNodes; ++j ) {
         if ( neighborNodes[j] == proxyNode ) break;
       }
       if ( j == nProxyNodes ) {
         neighborNodes[nProxyNodes] = proxyNode;
         nProxyNodes++;
       }
     }
   }
   return nProxyNodes;
 }

void LdbCoordinator::resume ( void )

Definition at line 715 of file LdbCoordinator.C.

References DebugM, initialize(), ldbCycleNum, PatchMap::Object(), Sync::Object(), ComputeMap::Object(), and Sync::openSync().

 {
   DebugM(3,"resume()\n");
   //  printLocalLdbReport();
 
   ldbCycleNum++;
   initialize(PatchMap::Object(),ComputeMap::Object(),1);
 
   Sync::Object()->openSync();
 }

void LdbCoordinator::resume2 ( void )

Definition at line 735 of file LdbCoordinator.C.

References awakenSequencers(), and DebugM.

 {
   DebugM(3,"resume2()\n");
 
 #if CONVERSE_VERSION_ELAN
   //  enableBlockingReceives();
 #endif
 
   awakenSequencers();
 }

void LdbCoordinator::ResumeFromSync ( void )

Definition at line 111 of file LdbCoordinator.C.

References myHandle, and theLbdb.

Referenced by LdbCoordinator().

 {
   theLbdb->DoneRegisteringObjects(myHandle);
   CkCallback cb(CkIndex_LdbCoordinator::nodeDone(NULL), 0, thisgroup);
   contribute(0, NULL, CkReduction::random, cb);
 }

void LdbCoordinator::resumeReady ( CkQdMsg * msg )

Definition at line 726 of file LdbCoordinator.C.

References DebugM, endi(), and iout.

                                              {
 
   iout << "LDB: =============== DONE WITH MIGRATION ================ " << CmiWallTimer() << "\n" << endi;
   DebugM(3,"resumeReady()\n");
   delete msg;
 
   CProxy_LdbCoordinator(thisgroup).resume2();
 }

void LdbCoordinator::sendCollectLoads ( CollectLoadsMsg * msg )

Definition at line 835 of file LdbCoordinator.C.

Referenced by Rebalancer::~Rebalancer().

                                                           {
   CProxy_LdbCoordinator(thisgroup)[0].collectLoads(msg);
 }

void LdbCoordinator::skipWork ( const LDObjHandle & handle )

inline

Definition at line 107 of file LdbCoordinator.h.

References nComputesReported.

Referenced by ComputeNonbondedPair::noWork(), ComputeNonbondedSelf::noWork(), and ComputeLCPO::noWork().

                                            {  // increment counter only
     nComputesReported++;
   }

void LdbCoordinator::startWork ( const LDObjHandle & handle )

inline

Definition at line 101 of file LdbCoordinator.h.

References theLbdb.

Referenced by ComputePatch::doWork(), ComputePatchPair::doWork(), ComputeLCPO::doWork(), ComputeHomeTuples< TholeElem, Thole, TholeValue >::doWork(), SimpleBroadcastObject< Tensor >::get(), and Sequencer::suspend().

                                             {  // start timer
     theLbdb->ObjectStart(handle);
   }

void LdbCoordinator::staticMigrateFn	(	LDObjHandle	handle,
		int	dest
	)

static

Definition at line 62 of file LdbCoordinator.C.

References Migrate(), and Object().

Referenced by LdbCoordinator().

 {
    LdbCoordinator::Object()->Migrate(handle,dest);
 }

void LdbCoordinator::staticQueryEstLoadFn ( LDOMHandle h )

static

Definition at line 86 of file LdbCoordinator.C.

Referenced by LdbCoordinator().

 {
   CkPrintf("I'm supposed to query load\n");
 }

void LdbCoordinator::staticReceiveAtSync ( void * data )

static

Definition at line 91 of file LdbCoordinator.C.

Referenced by LdbCoordinator().

 {
 
 #if CONVERSE_VERSION_ELAN
     //disableBlockingReceives();
 #endif
 
   ((LdbCoordinator*)data)->AtSyncBarrierReached();
 }

void LdbCoordinator::staticResumeFromSync ( void * data )

static

Definition at line 106 of file LdbCoordinator.C.

Referenced by LdbCoordinator().

 {
   ((LdbCoordinator*)data)->ResumeFromSync();
 }

void LdbCoordinator::staticStatsFn	(	LDOMHandle	h,
		int	state
	)

static

Definition at line 81 of file LdbCoordinator.C.

Referenced by LdbCoordinator().

 {
   CkPrintf("I'm supposed to set stats\n");
 }

void LdbCoordinator::updateComputesReady ( )

Definition at line 708 of file LdbCoordinator.C.

References DebugM.

Referenced by ExecuteMigrations(), and nodeDone().

                                          {
   DebugM(3,"updateComputesReady()\n");
 
   CProxy_LdbCoordinator(thisgroup).resume();
   CkStartQD(CkIndex_LdbCoordinator::resumeReady((CkQdMsg*)0),&thishandle);
 }