Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | File List | Class Members | File Members

LdbCoordinator Class Reference

#include <LdbCoordinator.h>

Inheritance diagram for LdbCoordinator:

BOCclass List of all members.

Public Member Functions

 LdbCoordinator ()
 ~LdbCoordinator (void)
void initialize (PatchMap *pmap, ComputeMap *cmap, int reinit=0)
void createLoadBalancer ()
void patchLoad (PatchID id, int nAtoms, int timestep)
void startWork (ComputeID id, int timestep)
void endWork (ComputeID id, int timestep)
void rebalance (Sequencer *seq, PatchID id)
void rebalance (Controller *seq)
void nodeDone (void)
void updateComputesReady ()
void barrier (void)
void resume (void)
void resumeReady (CkQdMsg *msg)
void resume2 (void)
int getNumStepsToRun (void)
void ReceiveAtSync (void)
void Migrate (LDObjHandle handle, int dest)
void RecvMigrate (LdbMigrateMsg *)
void ProcessMigrate (LdbMigrateMsg *)
void ExpectMigrate (LdbMigrateMsg *)
void ResumeFromSync (void)
void ExecuteMigrations (void)
void awakenSequencers (void)
int requiredProxies (PatchID id, int[])
void printRequiredProxies (PatchID id, FILE *fp)
void printLocalLdbReport (void)

Static Public Member Functions

LdbCoordinatorObject ()
void staticMigrateFn (LDObjHandle handle, int dest)
void staticStatsFn (LDOMHandle h, int state)
void staticQueryEstLoadFn (LDOMHandle h)
void staticReceiveAtSync (void *data)
void staticResumeFromSync (void *data)

Public Attributes

int stepsPerLdbCycle
int nLocalComputes
int nLocalPatches
int nPatchesReported
int nPatchesExpected
int nComputesReported
int nComputesExpected
int controllerReported
int controllerExpected
int nStatsMessagesReceived
int nStatsMessagesExpected
ComputeMapcomputeMap
PatchMappatchMap
int * patchNAtoms
int nPatches
ControllercontrollerThread
Sequencer ** sequencerThreads
int ldbCycleNum
int numStepsToRun
int firstLdbStep
int totalStepsDone
int takingLdbData
int nodesDone
FILE * ldbStatsFP
computeInfocomputeArray
patchInfopatchArray
processorInfoprocessorArray
LBDatabase * theLbdb
LDOMid myOMid
LDOMHandle myHandle
LDObjHandle * objHandles
int numComputes
int nRegisteredObjs
LDBarrierClient ldBarrierHandle
int reg_all_objs
LDObjHandle * patchHandles

Constructor & Destructor Documentation

LdbCoordinator::LdbCoordinator  ) 
 

Definition at line 135 of file LdbCoordinator.C.

References computeArray, CreateNamdCentLB(), iERROR(), iFILE, iout, iPE(), ldBarrierHandle, ldbCycleNum, ldbStatsFP, myHandle, myOMid, nLocalComputes, nLocalPatches, nodesDone, objHandles, patchArray, patchNAtoms, processorArray, reg_all_objs, sequencerThreads, staticQueryEstLoadFn(), staticReceiveAtSync(), staticResumeFromSync(), takingLdbData, theLbdb, and totalStepsDone.

00136 {
00137   if (CpvAccess(LdbCoordinator_instance) == NULL) {
00138     CpvAccess(LdbCoordinator_instance) = this;
00139   } else {
00140     iout << iFILE << iERROR << iPE 
00141          << "LdbCoordinator instanced twice on same node!" << endi;
00142     CkExit();
00143   }
00144   
00145 #if 0
00146   // Create a load balancer
00147   if (CkMyPe() == 0) {
00148     //   CreateCentralLB();
00149     CreateNamdCentLB();
00150     //   CreateNamdNborLB();
00151   }
00152 #endif
00153 
00154   ldbCycleNum = 1;
00155   takingLdbData = 1;
00156   totalStepsDone = 0;
00157   nLocalComputes = nLocalPatches = 0;
00158   patchNAtoms = (int *) NULL;
00159   sequencerThreads = (Sequencer **) NULL;
00160   ldbStatsFP = NULL;
00161   computeArray = NULL;
00162   patchArray = NULL;
00163   processorArray = NULL;
00164 
00165   nodesDone = 0;
00166 
00167   // Register self as an object manager for new charm++ balancer framework
00168 #if CHARM_VERSION > 050610
00169   theLbdb = LBDatabase::Object(); 
00170 #else
00171   theLbdb = CProxy_LBDatabase::ckLocalBranch(lbdb);
00172 #endif
00173 
00174   // Set the load balancing period (in seconds).  Without this the
00175   // load balancing framework will hang until 1 second has passed
00176   // since the last load balancing, causing hiccups in very fast runs.
00177   // Unfortunately, the clock is already set for the first load
00178   // balancing, but only +LBPeriod 1.0e-5 can fix that in older charm.
00179   // For newer versions this is handled in initproc above.
00180 
00181   theLbdb->SetLBPeriod(1.0e-5);
00182 
00183 #if CHARM_VERSION > 050403
00184   myOMid.id.idx = 1;
00185 #else
00186   myOMid.id = 1;
00187 #endif
00188   LDCallbacks cb = { (LDMigrateFn)staticMigrateFn,
00189                      (LDStatsFn)staticStatsFn,
00190                      (LDQueryEstLoadFn)staticQueryEstLoadFn
00191                    };
00192   myHandle = theLbdb->RegisterOM(myOMid,(void*)this,cb);
00193 
00194   // Add myself as a local barrier receiver, so I know when I might
00195   // be registering objects.
00196   theLbdb->AddLocalBarrierReceiver((LDBarrierFn)staticReceiveAtSync,
00197                                    (void*)this);;
00198 
00199   // Also, add a local barrier client, to trigger load balancing
00200   ldBarrierHandle = theLbdb->
00201     AddLocalBarrierClient((LDResumeFn)staticResumeFromSync,
00202                           (void*)this);
00203   objHandles = 0;
00204   reg_all_objs = 1;
00205 
00206 }

LdbCoordinator::~LdbCoordinator void   ) 
 

Definition at line 208 of file LdbCoordinator.C.

References ldbStatsFP.

00209 {
00210   delete [] patchNAtoms;
00211   delete [] sequencerThreads;
00212   delete [] objHandles;
00213   if (CkMyPe() == 0)
00214   {
00215     delete [] computeArray;
00216     delete [] patchArray;
00217     delete [] processorArray;
00218   }
00219   if (ldbStatsFP)
00220     fclose(ldbStatsFP);
00221 
00222 }


Member Function Documentation

void LdbCoordinator::awakenSequencers void   ) 
 

Definition at line 634 of file LdbCoordinator.C.

References Sequencer::awaken(), Controller::awaken(), controllerThread, PatchMap::numPatches(), patchMap, and sequencerThreads.

Referenced by resume2().

00635 {
00636   if (controllerThread)
00637   {
00638     controllerThread->awaken();
00639     controllerThread = NULL;
00640   }
00641   for(int i=0; i < patchMap->numPatches(); i++)
00642   {
00643     if (sequencerThreads[i])
00644     {
00645       sequencerThreads[i]->awaken();
00646     }
00647     sequencerThreads[i]= NULL;
00648   }
00649 }

void LdbCoordinator::barrier void   ) 
 

Definition at line 528 of file LdbCoordinator.C.

References controllerReported, ldBarrierHandle, NAMD_bug(), nComputesReported, nPatchesReported, and theLbdb.

00529 {
00530   if ( (nPatchesReported != nPatchesExpected) 
00531        || (nComputesReported != nComputesExpected)
00532        || (controllerReported != controllerExpected) )
00533   {
00534     NAMD_bug("Load balancer received wrong number of events.\n");
00535   }
00536 
00537   theLbdb->AtLocalBarrier(ldBarrierHandle);
00538 }

void LdbCoordinator::createLoadBalancer  ) 
 

Definition at line 224 of file LdbCoordinator.C.

References CreateNamdCentLB(), Node::Object(), Node::simParameters, and simParams.

Referenced by Node::startup().

00225 {
00226   if (CkMyPe()==0) CkPrintf("Measuring processor speeds...");
00227   const SimParameters *simParams = Node::Object()->simParameters;
00228   /*if (simParams->ldbStrategy == LDBSTRAT_ALGNBOR) 
00229     CreateNamdNborLB();
00230   else {*/
00231     CreateNamdCentLB();
00232   if (CkMyPe()==0) CkPrintf(" Done.\n");
00233 }

void LdbCoordinator::endWork ComputeID  id,
int  timestep
 

Definition at line 498 of file LdbCoordinator.C.

References nComputesReported, numComputes, objHandles, and theLbdb.

Referenced by ComputeNonbondedSelf::doForce(), ComputeNonbondedPair::doForce(), ComputeSelfTuples< CrosstermElem, Crossterm, CrosstermValue >::doWork(), and ComputeNonbondedPair::noWork().

00499 {
00500   CmiAssert(id >=0 && id < numComputes);
00501   theLbdb->ObjectStop(objHandles[id]);
00502   nComputesReported++;
00503 }

void LdbCoordinator::ExecuteMigrations void   ) 
 

Definition at line 550 of file LdbCoordinator.C.

References ComputeMgr::updateComputes(), and updateComputesReady().

Referenced by nodeDone().

00551 {
00552  // computeMgr->updateComputes() call only on Node(0) i.e. right here
00553   // This will barrier for all Nodes - (i.e. Computes must be
00554   // here and with proxies before anyone can start up
00555 
00556   CProxy_ComputeMgr cm(CpvAccess(BOCclass_group).computeMgr);
00557   ComputeMgr *computeMgr = cm.ckLocalBranch();
00558 #if CHARM_VERSION > 050402
00559   computeMgr->updateComputes(CkIndex_LdbCoordinator::
00560                              updateComputesReady(),thisgroup);
00561 #else
00562   computeMgr->updateComputes(CProxy_LdbCoordinator::
00563                              ckIdx_updateComputesReady(),thisgroup);
00564 #endif
00565 }

void LdbCoordinator::ExpectMigrate LdbMigrateMsg  ) 
 

Definition at line 584 of file LdbCoordinator.C.

References LdbMigrateMsg::handle, myHandle, objHandles, and theLbdb.

00585 {
00586   objHandles[m->handle.id.id[0]] 
00587     = theLbdb->RegisterObj(myHandle,m->handle.id,0,1);
00588 
00589   theLbdb->Migrated(objHandles[m->handle.id.id[0]]);
00590 
00591   delete m;
00592 }

int LdbCoordinator::getNumStepsToRun void   )  [inline]
 

Definition at line 64 of file LdbCoordinator.h.

Referenced by Sequencer::rebalanceLoad(), and Controller::rebalanceLoad().

00064 { return numStepsToRun; }

void LdbCoordinator::initialize PatchMap pmap,
ComputeMap cmap,
int  reinit = 0
 

Definition at line 235 of file LdbCoordinator.C.

References computeArray, computeMap, controllerExpected, controllerReported, controllerThread, CreateNamdCentLB(), CreateNamdNborLB(), SimParameters::firstLdbStep, firstLdbStep, iERROR(), iFILE, iout, iPE(), SimParameters::lastLdbStep, ldbCycleNum, SimParameters::ldbPeriod, SimParameters::ldbStrategy, myHandle, Node::myid(), NAMD_die(), nComputesExpected, nComputesReported, nLocalComputes, nLocalPatches, ComputeMap::node(), PatchMap::node(), nPatches, nPatchesExpected, nPatchesReported, nStatsMessagesExpected, nStatsMessagesReceived, ComputeMap::numComputes(), numComputes, PatchMap::numHomePatches(), Node::numNodes(), PatchMap::numPatches(), ComputeMap::numPids(), numStepsToRun, Node::Object(), objHandles, patchArray, patchHandles, patchMap, patchNAtoms, ComputeMap::pid(), processorArray, reg_all_objs, sequencerThreads, Node::simParameters, simParams, stepsPerLdbCycle, takingLdbData, theLbdb, totalStepsDone, and ComputeMap::type().

Referenced by resume(), and Node::startup().

00236 {
00237   const SimParameters *simParams = Node::Object()->simParameters;
00238 
00239 #if 0
00240   static int lbcreated = 0; // XXX static variables are unsafe for SMP
00241   // PE0 first time Create a load balancer
00242   if (CkMyPe() == 0 && !lbcreated) {
00243     if (simParams->ldbStrategy == LDBSTRAT_ALGNBOR) 
00244       CreateNamdNborLB();
00245     else {
00246       //   CreateCentralLB();
00247       CreateNamdCentLB();
00248     }
00249     lbcreated = 1;
00250   }
00251 #endif
00252 
00253   //  DebugM(10,"stepsPerLdbCycle initialized\n");
00254   stepsPerLdbCycle = simParams->ldbPeriod;
00255   firstLdbStep = simParams->firstLdbStep;
00256   int lastLdbStep = simParams->lastLdbStep;
00257 
00258   computeMap = cMap;
00259   patchMap = pMap;
00260 
00261   // Set the number of received messages correctly for node 0
00262 
00263   nStatsMessagesExpected = Node::Object()->numNodes();
00264   nStatsMessagesReceived = 0;
00265 
00266   if (patchNAtoms) 
00267     delete [] patchNAtoms;  // Depends on delete NULL to do nothing
00268   nPatches = patchMap->numPatches();
00269   patchNAtoms = new int[nPatches];
00270 
00271   typedef Sequencer *seqPtr;
00272 
00273   if ( ! reinit ) {
00274     delete [] sequencerThreads;  // Depends on delete NULL to do nothing
00275     sequencerThreads = new seqPtr[nPatches];
00276   }
00277 
00278   nLocalPatches=0;
00279 
00280   int i;
00281   for(i=0;i<nPatches;i++)
00282   {
00283     if (patchMap->node(i) == Node::Object()->myid())
00284     {
00285       nLocalPatches++;
00286       patchNAtoms[i]=0;
00287     } else {
00288       patchNAtoms[i]=-1;
00289     }
00290     if ( ! reinit ) sequencerThreads[i]=NULL;
00291   }
00292   if ( ! reinit ) controllerThread = NULL;
00293   if (nLocalPatches != patchMap->numHomePatches())
00294     NAMD_die("Disaggreement in patchMap data.\n");
00295  
00296   nLocalComputes = 0;
00297   numComputes = cMap->numComputes();
00298 
00299   for(i=0;i<numComputes;i++)  {
00300     if ( (computeMap->node(i) == Node::Object()->myid())
00301          && ( (computeMap->type(i) == computeNonbondedPairType)
00302               || (computeMap->type(i) == computeSelfBondsType)
00303               || (computeMap->type(i) == computeSelfAnglesType)
00304               || (computeMap->type(i) == computeSelfDihedralsType)
00305               || (computeMap->type(i) == computeSelfImpropersType)
00306               || (computeMap->type(i) == computeSelfCrosstermsType)
00307               || (computeMap->type(i) == computeNonbondedSelfType) ) ) {
00308       nLocalComputes++;
00309     }
00310   }
00311   
00312   // New LB frameworks registration
00313 
00314   // Allocate data structure to save incoming migrations.  Processor
00315   // zero will get all migrations
00316 
00317   // If this is the first time through, we need it register patches
00318   if (reg_all_objs) {
00319     // Tell the lbdb that I'm registering objects, until I'm done
00320     // registering them.
00321     theLbdb->RegisteringObjects(myHandle);
00322     
00323     patchHandles = new LDObjHandle[nLocalPatches];
00324     int patch_count=0;
00325     int i;
00326     for(i=0;i<nPatches;i++)
00327       if (patchMap->node(i) == Node::Object()->myid()) {
00328         LDObjid elemID;
00329         elemID.id[0] = i;
00330         elemID.id[1] = elemID.id[2] = elemID.id[3] = -2;
00331 
00332         if (patch_count >= nLocalPatches) {
00333           iout << iFILE << iERROR << iPE 
00334                << "LdbCoordinator found too many local patches!" << endi;
00335           CkExit();
00336         }
00337         patchHandles[patch_count] 
00338           = theLbdb->RegisterObj(myHandle,elemID,0,0);
00339         patch_count++;
00340       }
00341   
00342     // Allocate new object handles
00343     if (objHandles == 0) {
00344       objHandles = new LDObjHandle[numComputes];
00345       for(i=0; i<numComputes; i++)
00346         objHandles[i].id.id[0] = -1; // Use -1 to mark unused entries
00347 
00348       // Register computes
00349       for(i=0; i<numComputes; i++)  {
00350         if ( (computeMap->node(i) == Node::Object()->myid())
00351              && ( (computeMap->type(i) == computeNonbondedPairType)
00352                   || (computeMap->type(i) == computeSelfBondsType)
00353                   || (computeMap->type(i) == computeSelfAnglesType)
00354                   || (computeMap->type(i) == computeSelfDihedralsType)
00355                   || (computeMap->type(i) == computeSelfImpropersType)
00356                   || (computeMap->type(i) == computeSelfCrosstermsType)
00357                   || (computeMap->type(i) == computeNonbondedSelfType) ) ) {
00358           // Register the object with the load balancer
00359           // Store the depended patch IDs in the rest of the element ID
00360           LDObjid elemID;
00361           elemID.id[0] = i;
00362         
00363           if (cMap->numPids(i) > 2)
00364             elemID.id[3] = cMap->pid(i,2);
00365           else elemID.id[3] = -1;
00366 
00367           if (cMap->numPids(i) > 1)
00368             elemID.id[2] =  cMap->pid(i,1);
00369           else elemID.id[2] = -1;
00370 
00371           if (cMap->numPids(i) > 0)
00372             elemID.id[1] =  cMap->pid(i,0);
00373           else elemID.id[1] = -1;
00374 
00375           objHandles[i] = theLbdb->RegisterObj(myHandle,elemID,0,1);
00376         }
00377       }
00378     }
00379     theLbdb->DoneRegisteringObjects(myHandle);
00380     reg_all_objs = 0;
00381   }
00382 
00383   // Fixup to take care of the extra timestep at startup
00384   // This is pretty ugly here, but it makes the count correct
00385   
00386   // iout << "LDB Cycle Num: " << ldbCycleNum << "\n";
00387 
00388   if (ldbCycleNum==1)
00389   {
00390     totalStepsDone += firstLdbStep;
00391     numStepsToRun = firstLdbStep;
00392     takingLdbData = 0;
00393     theLbdb->CollectStatsOff();
00394   }
00395   else if ( (ldbCycleNum <= 4) || !takingLdbData )
00396   {
00397     totalStepsDone += firstLdbStep;
00398     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
00399       numStepsToRun = -1;
00400       takingLdbData = 0;
00401       theLbdb->CollectStatsOff();
00402     } else {
00403       numStepsToRun = firstLdbStep;
00404       takingLdbData = 1;
00405       theLbdb->CollectStatsOn();
00406     }
00407   }
00408   else 
00409   {
00410     totalStepsDone += stepsPerLdbCycle - firstLdbStep;
00411     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
00412       numStepsToRun = -1;
00413       takingLdbData = 0;
00414       theLbdb->CollectStatsOff();
00415     } else {
00416       numStepsToRun = stepsPerLdbCycle - firstLdbStep;
00417       takingLdbData = 0;
00418       theLbdb->CollectStatsOff();
00419     }
00420   }
00421 
00422 /*------------------------------------------------------------------------------*
00423  * ---------------------------------------------------------------------------- *
00424  * Comments inserted by Abhinav to clarify relation between ldbCycleNum,        *
00425  * load balancing step numbers (printed by the step() function) and             *
00426  * tracing of the steps                                                         *
00427  * ---------------------------------------------------------------------------- *
00428  * If trace is turned off in the beginning, then tracing is turned on           *
00429  * at ldbCycleNum = 4 and turned off at ldbCycleNum = 8. ldbCycleNum can        *
00430  * be adjusted by specifying firstLdbStep and ldbPeriod which are set by        *
00431  * default to 5*stepspercycle and 200*stepspercycle if not specified.           *
00432  *                                                                              *
00433  * If we choose firstLdbStep = 20 and ldbPeriod = 100, we have the              *
00434  * following timeline (for these particular numbers):                           *
00435  *                                                                              *
00436  * Tracing         :  <------ off ------><------------- on ------------><-- off *
00437  * Ldb Step() No   :              1     2     3        4      5        6      7 *
00438  * Iteration Steps : 00====20====40====60====80======160====180======260====280 *
00439  * ldbCycleNum     :  1     2     3     4     5        6      7        8      9 *
00440  * Instrumention   :          Inst  Inst  Inst           Inst            Inst   *
00441  * LDB Strategy    :             Alg7  Ref   Ref             Ref            Ref *
00442  *                                                                              *
00443  * Alg7 = AlgSeven                                                              *
00444  * Ref  = Refine (NamdCentLB.C, Rebalancer.C)                                   *
00445  * Inst = Instrumentation Phase (no real load balancing)                        *
00446  * ---------------------------------------------------------------------------- *
00447  *------------------------------------------------------------------------------*
00448  */
00449 
00450 #if CHARM_VERSION >= 050606
00451   if (traceAvailable()) {
00452     static int specialTracing = 0; // XXX static variables are unsafe for SMP
00453     if (ldbCycleNum == 1 && traceIsOn() == 0)  specialTracing = 1;
00454     if (specialTracing) {
00455       if (ldbCycleNum == 4) traceBegin();
00456       if (ldbCycleNum == 8) traceEnd();
00457     }
00458   }
00459 #endif
00460 
00461   nPatchesReported = 0;
00462   nPatchesExpected = nLocalPatches;
00463   nComputesReported = 0;
00464   nComputesExpected = nLocalComputes * numStepsToRun;
00465   controllerReported = 0;
00466   controllerExpected = ! CkMyPe();
00467 
00468   if (CkMyPe() == 0)
00469   {
00470     if (computeArray == NULL)
00471       computeArray = new computeInfo[numComputes];
00472     if (patchArray == NULL)
00473       patchArray = new patchInfo[nPatches];
00474     if (processorArray == NULL)
00475       processorArray = new processorInfo[CkNumPes()];
00476   }
00477     
00478   theLbdb->ClearLoads();
00479 }

void LdbCoordinator::Migrate LDObjHandle  handle,
int  dest
 

Definition at line 68 of file LdbCoordinator.C.

References LdbMigrateMsg::from, LdbMigrateMsg::handle, and LdbMigrateMsg::to.

Referenced by staticMigrateFn().

00069 {
00070   LdbMigrateMsg* msg = new LdbMigrateMsg;
00071   msg->handle = handle;
00072   msg->from = CkMyPe();
00073   msg->to = dest;
00074 #if CHARM_VERSION > 050402
00075   CProxy_LdbCoordinator ldbProxy(thisgroup);
00076   ldbProxy[CkMyPe()].RecvMigrate(msg);
00077 #else
00078   CProxy_LdbCoordinator(thisgroup).RecvMigrate(msg,CkMyPe());
00079 #endif
00080 }

void LdbCoordinator::nodeDone void   ) 
 

Definition at line 540 of file LdbCoordinator.C.

References ExecuteMigrations(), nodesDone, Node::numNodes(), and Node::Object().

00541 {
00542   nodesDone++;
00543 
00544   if (nodesDone==Node::Object()->numNodes()) {
00545     nodesDone=0;
00546     ExecuteMigrations();
00547   }
00548 }

LdbCoordinator* LdbCoordinator::Object  )  [inline, static]
 

Definition at line 47 of file LdbCoordinator.h.

Referenced by ComputeNonbondedSelf::doForce(), ComputeNonbondedPair::doForce(), ComputeSelfTuples< CrosstermElem, Crossterm, CrosstermValue >::doWork(), ComputeNonbondedPair::noWork(), Sequencer::rebalanceLoad(), Controller::rebalanceLoad(), Sequencer::Sequencer(), Node::startup(), and HomePatch::submitLoadStats().

00047                                    { 
00048     return CpvAccess(LdbCoordinator_instance); 
00049   }

void LdbCoordinator::patchLoad PatchID  id,
int  nAtoms,
int  timestep
 

Definition at line 481 of file LdbCoordinator.C.

References DebugM, nPatches, nPatchesReported, and patchNAtoms.

Referenced by HomePatch::submitLoadStats().

00482 {
00483   CmiAssert( id >=0 && id < nPatches);
00484   if (patchNAtoms[id] != -1) {
00485     patchNAtoms[id] = nAtoms;
00486     nPatchesReported++;
00487   } else {
00488     DebugM(10, "::patchLoad() Unexpected patch reporting in\n");
00489   }
00490 }

void LdbCoordinator::printLocalLdbReport void   ) 
 

Definition at line 694 of file LdbCoordinator.C.

References PatchMap::numPatches(), patchMap, and patchNAtoms.

00695 {
00696   char outputBuf[255];
00697   char *curLoc;
00698 
00699   CkPrintf("%d:Patch report:\n",CkMyPe());
00700   
00701   curLoc = outputBuf;
00702   int i,j=0;
00703   for(i=0; i<patchMap->numPatches(); i++)
00704   {
00705     if (patchNAtoms[i] != -1)
00706     {
00707       curLoc += sprintf(curLoc,"%5d: %5d ",i,patchNAtoms[i]);
00708       j++;
00709     } 
00710     if (((j % 4) == 0) && j)
00711     {
00712       curLoc = outputBuf;
00713       CkPrintf("[%d]%s\n",CkMyPe(),outputBuf);
00714       j=0;
00715     }
00716   }
00717 
00718   CkPrintf("%d:Compute report:\n",CkMyPe());
00719   
00720   curLoc = outputBuf;
00721   j=0;
00722 }

void LdbCoordinator::printRequiredProxies PatchID  id,
FILE *  fp
 

Definition at line 724 of file LdbCoordinator.C.

References requiredProxies().

00725 {
00726   // Check all two-away neighbors.
00727   // This is really just one-away neighbors, since 
00728   // two-away always returns zero: RKB
00729   int neighborNodes[PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
00730   const int nProxyNodes = requiredProxies(id,neighborNodes);
00731 
00732   fprintf(fp,"%4d ",nProxyNodes);
00733 
00734   for(int i=0;i<nProxyNodes;i++)
00735     fprintf(fp,"%4d ",neighborNodes[i]);
00736 }

void LdbCoordinator::ProcessMigrate LdbMigrateMsg  ) 
 

void LdbCoordinator::rebalance Controller seq  ) 
 

Definition at line 514 of file LdbCoordinator.C.

References controllerReported, controllerThread, DebugM, SimParameters::ldbStrategy, Node::Object(), and Node::simParameters.

00515 {
00516   if (Node::Object()->simParameters->ldbStrategy == LDBSTRAT_NONE)
00517     return;
00518 
00519   DebugM(3, "Controller reached load balance barrier.\n");
00520   controllerReported = 1;
00521   controllerThread = c;
00522 
00523   CProxy_LdbCoordinator(thisgroup).barrier();
00524 
00525   CthSuspend();
00526 }

void LdbCoordinator::rebalance Sequencer seq,
PatchID  id
 

Definition at line 505 of file LdbCoordinator.C.

References SimParameters::ldbStrategy, Node::Object(), sequencerThreads, Node::simParameters, and Sequencer::suspend().

Referenced by Sequencer::rebalanceLoad(), and Controller::rebalanceLoad().

00506 {
00507   if (Node::Object()->simParameters->ldbStrategy == LDBSTRAT_NONE)
00508     return;
00509 
00510   sequencerThreads[pid] = seq;
00511   seq->suspend();
00512 }

void LdbCoordinator::ReceiveAtSync void   ) 
 

Definition at line 102 of file LdbCoordinator.C.

References myHandle, and theLbdb.

00103 {
00104   theLbdb->RegisteringObjects(myHandle);
00105 }

void LdbCoordinator::RecvMigrate LdbMigrateMsg  ) 
 

Definition at line 567 of file LdbCoordinator.C.

References LdbMigrateMsg::handle, objHandles, theLbdb, and LdbMigrateMsg::to.

00568 {
00569   // This method receives the migration from the framework,
00570   // unregisters it, and sends it to the destination PE
00571   const int id = m->handle.id.id[0];
00572 
00573   theLbdb->UnregisterObj(objHandles[id]);
00574   objHandles[id].id.id[0] = -1;
00575 
00576 #if CHARM_VERSION > 050402
00577   CProxy_LdbCoordinator  ldbProxy(thisgroup);
00578   ldbProxy[m->to].ExpectMigrate(m);
00579 #else
00580   CProxy_LdbCoordinator(thisgroup).ExpectMigrate(m,m->to);
00581 #endif
00582 }

int LdbCoordinator::requiredProxies PatchID  id,
int  []
 

Definition at line 655 of file LdbCoordinator.C.

References PatchMap::basenode(), PatchMap::downstreamNeighbors(), PatchMap::node(), PatchID, and patchMap.

Referenced by printRequiredProxies().

00656 {
00657   enum proxyHere { No, Yes };
00658   int numNodes = CkNumPes();
00659   proxyHere *proxyNodes = new proxyHere[numNodes];
00660   int nProxyNodes;
00661   int i;
00662 
00663   // Note all home patches.
00664   for ( i = 0; i < numNodes; ++i )
00665   {
00666     proxyNodes[i] = No;
00667   }
00668   nProxyNodes=0;
00669 
00670   // Check all two-away neighbors.
00671   // This is really just one-away neighbors, since 
00672   // two-away always returns zero: RKB
00673   PatchID neighbors[1 + PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
00674 
00675   int myNode = patchMap->node(id);
00676   neighbors[0] = id;
00677   int numNeighbors = 1 + patchMap->downstreamNeighbors(id,neighbors+1);
00678   for ( i = 0; i < numNeighbors; ++i )
00679   {
00680     const int proxyNode = patchMap->basenode(neighbors[i]);
00681     if (proxyNode != myNode)
00682       if (proxyNodes[proxyNode] == No)
00683       {
00684         proxyNodes[proxyNode] = Yes;
00685         neighborNodes[nProxyNodes] = proxyNode;
00686         nProxyNodes++;
00687       }
00688   }
00689 
00690   delete [] proxyNodes;
00691   return nProxyNodes;
00692 }

void LdbCoordinator::resume void   ) 
 

Definition at line 605 of file LdbCoordinator.C.

References DebugM, initialize(), and ldbCycleNum.

00606 {
00607   DebugM(3,"resume()\n");
00608 
00609   //  printLocalLdbReport();
00610 
00611   ldbCycleNum++;
00612   initialize(PatchMap::Object(),ComputeMap::Object(),1);
00613 }

void LdbCoordinator::resume2 void   ) 
 

Definition at line 623 of file LdbCoordinator.C.

References awakenSequencers(), and DebugM.

00624 {
00625   DebugM(3,"resume2()\n");
00626 
00627 #if CONVERSE_VERSION_ELAN
00628   //  enableBlockingReceives();
00629 #endif
00630 
00631   awakenSequencers();
00632 }

void LdbCoordinator::ResumeFromSync void   ) 
 

Definition at line 112 of file LdbCoordinator.C.

References ResizeArrayIter< T >::begin(), ResizeArrayIter< T >::end(), PatchMap::homePatchList(), HomePatchList, myHandle, PatchMap::Object(), and theLbdb.

00113 {
00114   theLbdb->DoneRegisteringObjects(myHandle);
00115 #if CMK_PERSISTENT_COMM
00116   if (takingLdbData) {
00117 //CmiPrintf("[%d] CmiDestoryPersistent\n", CkMyPe());
00118 
00119     HomePatchList *hpl = PatchMap::Object()->homePatchList();
00120     ResizeArrayIter<HomePatchElem> ai(*hpl);
00121     for (ai=ai.begin(); ai != ai.end(); ai++) {
00122       HomePatch *patch = (*ai).patch;
00123       patch->destoryPersistComm();
00124     }
00125   }
00126 #endif
00127   CProxy_LdbCoordinator cl(thisgroup);
00128 #if CHARM_VERSION > 050402
00129   cl[0].nodeDone();
00130 #else
00131   cl.nodeDone(0);
00132 #endif
00133 }

void LdbCoordinator::resumeReady CkQdMsg *  msg  ) 
 

Definition at line 615 of file LdbCoordinator.C.

References DebugM.

00615                                              {
00616 
00617   DebugM(3,"resumeReady()\n");
00618   delete msg;
00619 
00620   CProxy_LdbCoordinator(thisgroup).resume2();
00621 }

void LdbCoordinator::startWork ComputeID  id,
int  timestep
 

Definition at line 492 of file LdbCoordinator.C.

References numComputes, objHandles, and theLbdb.

Referenced by ComputeNonbondedSelf::doForce(), ComputeNonbondedPair::doForce(), ComputeSelfTuples< CrosstermElem, Crossterm, CrosstermValue >::doWork(), and ComputeNonbondedPair::noWork().

00493 {
00494   CmiAssert(id >=0 && id < numComputes);
00495   theLbdb->ObjectStart(objHandles[id]);
00496 }

void LdbCoordinator::staticMigrateFn LDObjHandle  handle,
int  dest
[static]
 

Definition at line 58 of file LdbCoordinator.C.

References Migrate().

00059 {
00060 #if CHARM_VERSION > 050606
00061    LdbCoordinator *ldbCoordinator = (LdbCoordinator *)LDOMUserData(handle.omhandle);
00062    ldbCoordinator->Migrate(handle,dest);
00063 #else
00064    ((LdbCoordinator*)handle.omhandle.user_ptr)->Migrate(handle,dest);
00065 #endif
00066 }

void LdbCoordinator::staticQueryEstLoadFn LDOMHandle  h  )  [static]
 

Definition at line 87 of file LdbCoordinator.C.

Referenced by LdbCoordinator().

00088 {
00089   CkPrintf("I'm supposed to query load\n");
00090 }

void LdbCoordinator::staticReceiveAtSync void *  data  )  [static]
 

Definition at line 92 of file LdbCoordinator.C.

Referenced by LdbCoordinator().

00093 {
00094 
00095 #if CONVERSE_VERSION_ELAN
00096     //disableBlockingReceives();
00097 #endif
00098 
00099   ((LdbCoordinator*)data)->ReceiveAtSync();
00100 }

void LdbCoordinator::staticResumeFromSync void *  data  )  [static]
 

Definition at line 107 of file LdbCoordinator.C.

Referenced by LdbCoordinator().

00108 {
00109   ((LdbCoordinator*)data)->ResumeFromSync();
00110 }

void LdbCoordinator::staticStatsFn LDOMHandle  h,
int  state
[static]
 

Definition at line 82 of file LdbCoordinator.C.

00083 {
00084   CkPrintf("I'm supposed to set stats\n");
00085 }

void LdbCoordinator::updateComputesReady  ) 
 

Definition at line 594 of file LdbCoordinator.C.

References DebugM.

Referenced by ExecuteMigrations().

00594                                          {
00595   DebugM(3,"updateComputesReady()\n");
00596 
00597   CProxy_LdbCoordinator(thisgroup).resume();
00598 #if CHARM_VERSION > 050402
00599   CkStartQD(CkIndex_LdbCoordinator::resumeReady((CkQdMsg*)0),&thishandle);
00600 #else
00601   CkStartQD(CProxy_LdbCoordinator::ckIdx_resumeReady((CkQdMsg*)0),&thishandle);
00602 #endif
00603 }


Member Data Documentation

computeInfo* LdbCoordinator::computeArray
 

Definition at line 112 of file LdbCoordinator.h.

Referenced by initialize(), and LdbCoordinator().

ComputeMap* LdbCoordinator::computeMap
 

Definition at line 95 of file LdbCoordinator.h.

Referenced by initialize().

int LdbCoordinator::controllerExpected
 

Definition at line 92 of file LdbCoordinator.h.

Referenced by initialize().

int LdbCoordinator::controllerReported
 

Definition at line 91 of file LdbCoordinator.h.

Referenced by barrier(), initialize(), and rebalance().

Controller* LdbCoordinator::controllerThread
 

Definition at line 99 of file LdbCoordinator.h.

Referenced by awakenSequencers(), initialize(), and rebalance().

int LdbCoordinator::firstLdbStep
 

Definition at line 105 of file LdbCoordinator.h.

Referenced by initialize().

LDBarrierClient LdbCoordinator::ldBarrierHandle
 

Definition at line 121 of file LdbCoordinator.h.

Referenced by barrier(), and LdbCoordinator().

int LdbCoordinator::ldbCycleNum
 

Definition at line 102 of file LdbCoordinator.h.

Referenced by initialize(), LdbCoordinator(), and resume().

FILE* LdbCoordinator::ldbStatsFP
 

Definition at line 111 of file LdbCoordinator.h.

Referenced by LdbCoordinator(), and ~LdbCoordinator().

LDOMHandle LdbCoordinator::myHandle
 

Definition at line 117 of file LdbCoordinator.h.

Referenced by ExpectMigrate(), initialize(), LdbCoordinator(), ReceiveAtSync(), and ResumeFromSync().

LDOMid LdbCoordinator::myOMid
 

Definition at line 116 of file LdbCoordinator.h.

Referenced by LdbCoordinator().

int LdbCoordinator::nComputesExpected
 

Definition at line 90 of file LdbCoordinator.h.

Referenced by initialize().

int LdbCoordinator::nComputesReported
 

Definition at line 89 of file LdbCoordinator.h.

Referenced by barrier(), endWork(), and initialize().

int LdbCoordinator::nLocalComputes