NAMD: LdbCoordinator Class Reference

00122 {
00123   if (CkpvAccess(LdbCoordinator_instance) == NULL) {
00124     CkpvAccess(LdbCoordinator_instance) = this;
00125   } else {
00126     iout << iFILE << iERROR << iPE 
00127          << "LdbCoordinator instanced twice on same node!" << endi;
00128     CkExit();
00129   }
00130   
00131 #if 0
00132   // Create a load balancer
00133   if (CkMyPe() == 0) {
00134     //   CreateCentralLB();
00135     CreateNamdCentLB();
00136     //   CreateNamdNborLB();
00137   }
00138 #endif
00139 
00140   ldbCycleNum = 1;
00141   takingLdbData = 1;
00142   totalStepsDone = 0;
00143   nLocalComputes = nLocalPatches = 0;
00144   patchNAtoms = (int *) NULL;
00145   sequencerThreads = (Sequencer **) NULL;
00146   ldbStatsFP = NULL;
00147   computeArray = NULL;
00148   patchArray = NULL;
00149   processorArray = NULL;
00150 
00151   nodesDone = 0;
00152 
00153   // Register self as an object manager for new charm++ balancer framework
00154   theLbdb = LBDatabase::Object(); 
00155 
00156   // Set the load balancing period (in seconds).  Without this the
00157   // load balancing framework will hang until 1 second has passed
00158   // since the last load balancing, causing hiccups in very fast runs.
00159   // Unfortunately, the clock is already set for the first load
00160   // balancing, but only +LBPeriod 1.0e-5 can fix that in older charm.
00161   // For newer versions this is handled in initproc above.
00162 
00163   theLbdb->SetLBPeriod(1.0e-5);
00164 
00165   myOMid.id.idx = 1;
00166   LDCallbacks cb = { (LDMigrateFn)staticMigrateFn,
00167                      (LDStatsFn)staticStatsFn,
00168                      (LDQueryEstLoadFn)staticQueryEstLoadFn
00169                    };
00170   myHandle = theLbdb->RegisterOM(myOMid,(void*)this,cb);
00171 
00172   // Add myself as a local barrier receiver, so I know when I might
00173   // be registering objects.
00174   theLbdb->AddLocalBarrierReceiver((LDBarrierFn)staticReceiveAtSync,
00175                                    (void*)this);;
00176 
00177   // Also, add a local barrier client, to trigger load balancing
00178   ldBarrierHandle = theLbdb->
00179     AddLocalBarrierClient((LDResumeFn)staticResumeFromSync,
00180                           (void*)this);
00181   objHandles = 0;
00182   reg_all_objs = 1;
00183 
00184 }

00187 {
00188   delete [] patchNAtoms;
00189   delete [] sequencerThreads;
00190   delete [] objHandles;
00191   if (CkMyPe() == 0)
00192   {
00193     delete [] computeArray;
00194     delete [] patchArray;
00195     delete [] processorArray;
00196   }
00197   if (ldbStatsFP)
00198     fclose(ldbStatsFP);
00199 
00200 }

00612 {
00613   if (controllerThread)
00614   {
00615     controllerThread->awaken();
00616     controllerThread = NULL;
00617   }
00618   for(int i=0; i < patchMap->numPatches(); i++)
00619   {
00620     if (sequencerThreads[i])
00621     {
00622       sequencerThreads[i]->awaken();
00623     }
00624     sequencerThreads[i]= NULL;
00625   }
00626 }

00518 {
00519   if ( (nPatchesReported != nPatchesExpected) 
00520        || (nComputesReported != nComputesExpected)
00521        || (controllerReported != controllerExpected) )
00522   {
00523     NAMD_bug("Load balancer received wrong number of events.\n");
00524   }
00525 
00526   theLbdb->AtLocalBarrier(ldBarrierHandle);
00527 }

00203 {
00204   if (CkMyPe()==0) 
00205     iout << "LDB: Measuring processor speeds ..." << endi;
00206   const SimParameters *simParams = Node::Object()->simParameters;
00207   /*if (simParams->ldbStrategy == LDBSTRAT_ALGNBOR) 
00208     CreateNamdNborLB();
00209   else {*/
00210     CreateNamdCentLB();
00211   if (CkMyPe()==0)
00212     iout << " Done.\n" << endi;
00213 }

00487 {
00488   CmiAssert(id >=0 && id < numComputes);
00489   theLbdb->ObjectStop(objHandles[id]);
00490   nComputesReported++;
00491 }

00541 {
00542  // computeMgr->updateComputes() call only on Node(0) i.e. right here
00543   // This will barrier for all Nodes - (i.e. Computes must be
00544   // here and with proxies before anyone can start up
00545 
00546   CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00547   ComputeMgr *computeMgr = cm.ckLocalBranch();
00548   computeMgr->updateComputes(CkIndex_LdbCoordinator::
00549                              updateComputesReady(),thisgroup);
00550 }

00566 {
00567   objHandles[m->handle.id.id[0]] 
00568     = theLbdb->RegisterObj(myHandle,m->handle.id,0,1);
00569 
00570   theLbdb->Migrated(objHandles[m->handle.id.id[0]]);
00571 
00572   delete m;
00573 }

00064 { return numStepsToRun; }

00216 {
00217   const SimParameters *simParams = Node::Object()->simParameters;
00218 
00219 #if 0
00220   static int lbcreated = 0; // XXX static variables are unsafe for SMP
00221   // PE0 first time Create a load balancer
00222   if (CkMyPe() == 0 && !lbcreated) {
00223     if (simParams->ldbStrategy == LDBSTRAT_ALGNBOR) 
00224       CreateNamdNborLB();
00225     else {
00226       //   CreateCentralLB();
00227       CreateNamdCentLB();
00228     }
00229     lbcreated = 1;
00230   }
00231 #endif
00232 
00233   //  DebugM(10,"stepsPerLdbCycle initialized\n");
00234   stepsPerLdbCycle = simParams->ldbPeriod;
00235   firstLdbStep = simParams->firstLdbStep;
00236   int lastLdbStep = simParams->lastLdbStep;
00237 
00238   computeMap = cMap;
00239   patchMap = pMap;
00240 
00241   // Set the number of received messages correctly for node 0
00242 
00243   nStatsMessagesExpected = Node::Object()->numNodes();
00244   nStatsMessagesReceived = 0;
00245 
00246   if (patchNAtoms) 
00247     delete [] patchNAtoms;  // Depends on delete NULL to do nothing
00248   nPatches = patchMap->numPatches();
00249   patchNAtoms = new int[nPatches];
00250 
00251   typedef Sequencer *seqPtr;
00252 
00253   if ( ! reinit ) {
00254     delete [] sequencerThreads;  // Depends on delete NULL to do nothing
00255     sequencerThreads = new seqPtr[nPatches];
00256   }
00257 
00258   nLocalPatches=0;
00259 
00260   int i;
00261   for(i=0;i<nPatches;i++)
00262   {
00263     if (patchMap->node(i) == Node::Object()->myid())
00264     {
00265       nLocalPatches++;
00266       patchNAtoms[i]=0;
00267     } else {
00268       patchNAtoms[i]=-1;
00269     }
00270     if ( ! reinit ) sequencerThreads[i]=NULL;
00271   }
00272   if ( ! reinit ) controllerThread = NULL;
00273   if (nLocalPatches != patchMap->numHomePatches())
00274     NAMD_die("Disaggreement in patchMap data.\n");
00275  
00276   nLocalComputes = 0;
00277   numComputes = cMap->numComputes();
00278 
00279   for(i=0;i<numComputes;i++)  {
00280     if ( (computeMap->node(i) == Node::Object()->myid())
00281          && ( 0
00282 #ifndef NAMD_CUDA
00283               || (computeMap->type(i) == computeNonbondedSelfType)
00284               || (computeMap->type(i) == computeNonbondedPairType)
00285 #endif
00286               || (computeMap->type(i) == computeSelfExclsType)
00287               || (computeMap->type(i) == computeSelfBondsType)
00288               || (computeMap->type(i) == computeSelfAnglesType)
00289               || (computeMap->type(i) == computeSelfDihedralsType)
00290               || (computeMap->type(i) == computeSelfImpropersType)
00291               || (computeMap->type(i) == computeSelfCrosstermsType)
00292         ) ) {
00293       nLocalComputes++;
00294     }
00295   }
00296   
00297   // New LB frameworks registration
00298 
00299   // Allocate data structure to save incoming migrations.  Processor
00300   // zero will get all migrations
00301 
00302   // If this is the first time through, we need it register patches
00303   if (reg_all_objs) {
00304     // Tell the lbdb that I'm registering objects, until I'm done
00305     // registering them.
00306     theLbdb->RegisteringObjects(myHandle);
00307     
00308     patchHandles = new LDObjHandle[nLocalPatches];
00309     int patch_count=0;
00310     int i;
00311     for(i=0;i<nPatches;i++)
00312       if (patchMap->node(i) == Node::Object()->myid()) {
00313         LDObjid elemID;
00314         elemID.id[0] = i;
00315         elemID.id[1] = elemID.id[2] = elemID.id[3] = -2;
00316 
00317         if (patch_count >= nLocalPatches) {
00318           iout << iFILE << iERROR << iPE 
00319                << "LdbCoordinator found too many local patches!" << endi;
00320           CkExit();
00321         }
00322         patchHandles[patch_count] 
00323           = theLbdb->RegisterObj(myHandle,elemID,0,0);
00324         patch_count++;
00325       }
00326   
00327     // Allocate new object handles
00328     if (objHandles == 0) {
00329       objHandles = new LDObjHandle[numComputes];
00330       for(i=0; i<numComputes; i++)
00331         objHandles[i].id.id[0] = -1; // Use -1 to mark unused entries
00332 
00333       // Register computes
00334       for(i=0; i<numComputes; i++)  {
00335         if ( (computeMap->node(i) == Node::Object()->myid())
00336              && ( 0
00337 #ifndef NAMD_CUDA
00338                   || (computeMap->type(i) == computeNonbondedSelfType)
00339                   || (computeMap->type(i) == computeNonbondedPairType)
00340 #endif
00341                   || (computeMap->type(i) == computeSelfExclsType)
00342                   || (computeMap->type(i) == computeSelfBondsType)
00343                   || (computeMap->type(i) == computeSelfAnglesType)
00344                   || (computeMap->type(i) == computeSelfDihedralsType)
00345                   || (computeMap->type(i) == computeSelfImpropersType)
00346                   || (computeMap->type(i) == computeSelfCrosstermsType)
00347                 ) ) {
00348           // Register the object with the load balancer
00349           // Store the depended patch IDs in the rest of the element ID
00350           LDObjid elemID;
00351           elemID.id[0] = i;
00352         
00353           if (cMap->numPids(i) > 2)
00354             elemID.id[3] = cMap->pid(i,2);
00355           else elemID.id[3] = -1;
00356 
00357           if (cMap->numPids(i) > 1)
00358             elemID.id[2] =  cMap->pid(i,1);
00359           else elemID.id[2] = -1;
00360 
00361           if (cMap->numPids(i) > 0)
00362             elemID.id[1] =  cMap->pid(i,0);
00363           else elemID.id[1] = -1;
00364 
00365           objHandles[i] = theLbdb->RegisterObj(myHandle,elemID,0,1);
00366         }
00367       }
00368     }
00369     theLbdb->DoneRegisteringObjects(myHandle);
00370     reg_all_objs = 0;
00371   }
00372 
00373   // Fixup to take care of the extra timestep at startup
00374   // This is pretty ugly here, but it makes the count correct
00375   
00376   // iout << "LDB Cycle Num: " << ldbCycleNum << "\n";
00377 
00378   if (ldbCycleNum==1)
00379   {
00380     totalStepsDone += firstLdbStep;
00381     numStepsToRun = firstLdbStep;
00382     takingLdbData = 0;
00383     theLbdb->CollectStatsOff();
00384   }
00385   else if ( (ldbCycleNum <= 4) || !takingLdbData )
00386   {
00387     totalStepsDone += firstLdbStep;
00388     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
00389       numStepsToRun = -1;
00390       takingLdbData = 0;
00391       theLbdb->CollectStatsOff();
00392     } else {
00393       numStepsToRun = firstLdbStep;
00394       takingLdbData = 1;
00395       theLbdb->CollectStatsOn();
00396     }
00397   }
00398   else 
00399   {
00400     totalStepsDone += stepsPerLdbCycle - firstLdbStep;
00401     if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
00402       numStepsToRun = -1;
00403       takingLdbData = 0;
00404       theLbdb->CollectStatsOff();
00405     } else {
00406       numStepsToRun = stepsPerLdbCycle - firstLdbStep;
00407       takingLdbData = 0;
00408       theLbdb->CollectStatsOff();
00409     }
00410   }
00411 
00412 /*------------------------------------------------------------------------------*
00413  * ---------------------------------------------------------------------------- *
00414  * Comments inserted by Abhinav to clarify relation between ldbCycleNum,        *
00415  * load balancing step numbers (printed by the step() function) and             *
00416  * tracing of the steps                                                         *
00417  * ---------------------------------------------------------------------------- *
00418  * If trace is turned off in the beginning, then tracing is turned on           *
00419  * at ldbCycleNum = 4 and turned off at ldbCycleNum = 8. ldbCycleNum can        *
00420  * be adjusted by specifying firstLdbStep and ldbPeriod which are set by        *
00421  * default to 5*stepspercycle and 200*stepspercycle if not specified.           *
00422  *                                                                              *
00423  * If we choose firstLdbStep = 20 and ldbPeriod = 100, we have the              *
00424  * following timeline (for these particular numbers):                           *
00425  *                                                                              *
00426  * Tracing         :  <------ off ------><------------- on ------------><-- off *
00427  * Ldb Step() No   :              1     2     3        4      5        6      7 *
00428  * Iteration Steps : 00====20====40====60====80======160====180======260====280 *
00429  * ldbCycleNum     :  1     2     3     4     5        6      7        8      9 *
00430  * Instrumention   :          Inst  Inst  Inst           Inst            Inst   *
00431  * LDB Strategy    :             Alg7  Ref   Ref             Ref            Ref *
00432  *                                                                              *
00433  * Alg7 = AlgSeven                                                              *
00434  * Ref  = Refine (NamdCentLB.C, Rebalancer.C)                                   *
00435  * Inst = Instrumentation Phase (no real load balancing)                        *
00436  * ---------------------------------------------------------------------------- *
00437  *------------------------------------------------------------------------------*
00438  */
00439 
00440   if (traceAvailable()) {
00441     static int specialTracing = 0; // XXX static variables are unsafe for SMP
00442     if (ldbCycleNum == 1 && traceIsOn() == 0)  specialTracing = 1;
00443     if (specialTracing) {
00444       if (ldbCycleNum == 4) traceBegin();
00445       if (ldbCycleNum == 8) traceEnd();
00446     }
00447   }
00448 
00449   nPatchesReported = 0;
00450   nPatchesExpected = nLocalPatches;
00451   nComputesReported = 0;
00452   nComputesExpected = nLocalComputes * numStepsToRun;
00453   controllerReported = 0;
00454   controllerExpected = ! CkMyPe();
00455 
00456   if (CkMyPe() == 0)
00457   {
00458     if (computeArray == NULL)
00459       computeArray = new computeInfo[numComputes];
00460     if (patchArray == NULL)
00461       patchArray = new patchInfo[nPatches];
00462     if (processorArray == NULL)
00463       processorArray = new processorInfo[CkNumPes()];
00464   }
00465     
00466   theLbdb->ClearLoads();
00467 }

00063 {
00064   LdbMigrateMsg* msg = new LdbMigrateMsg;
00065   msg->handle = handle;
00066   msg->from = CkMyPe();
00067   msg->to = dest;
00068   CProxy_LdbCoordinator ldbProxy(thisgroup);
00069   ldbProxy[CkMyPe()].RecvMigrate(msg);
00070 }

00530 {
00531   nodesDone++;
00532 
00533   if (nodesDone==Node::Object()->numNodes()) {
00534     iout << "LDB: ============== END OF LOAD BALANCING =============== " << CmiWallTimer() << "\n\n" << endi;
00535     nodesDone=0;
00536     ExecuteMigrations();
00537   }
00538 }

00047                                    { 
00048     return CkpvAccess(LdbCoordinator_instance); 
00049   }

00470 {
00471   CmiAssert( id >=0 && id < nPatches);
00472   if (patchNAtoms[id] != -1) {
00473     patchNAtoms[id] = nAtoms;
00474     nPatchesReported++;
00475   } else {
00476     DebugM(10, "::patchLoad() Unexpected patch reporting in\n");
00477   }
00478 }

00672 {
00673   char outputBuf[255];
00674   char *curLoc;
00675 
00676   CkPrintf("%d:Patch report:\n",CkMyPe());
00677   
00678   curLoc = outputBuf;
00679   int i,j=0;
00680   for(i=0; i<patchMap->numPatches(); i++)
00681   {
00682     if (patchNAtoms[i] != -1)
00683     {
00684       curLoc += sprintf(curLoc,"%5d: %5d ",i,patchNAtoms[i]);
00685       j++;
00686     } 
00687     if (((j % 4) == 0) && j)
00688     {
00689       curLoc = outputBuf;
00690       CkPrintf("[%d]%s\n",CkMyPe(),outputBuf);
00691       j=0;
00692     }
00693   }
00694 
00695   CkPrintf("%d:Compute report:\n",CkMyPe());
00696   
00697   curLoc = outputBuf;
00698   j=0;
00699 }

00702 {
00703   // Check all two-away neighbors.
00704   // This is really just one-away neighbors, since 
00705   // two-away always returns zero: RKB
00706   int neighborNodes[PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
00707   const int nProxyNodes = requiredProxies(id,neighborNodes);
00708 
00709   fprintf(fp,"%4d ",nProxyNodes);
00710 
00711   for(int i=0;i<nProxyNodes;i++)
00712     fprintf(fp,"%4d ",neighborNodes[i]);
00713 }

00503 {
00504   if (Node::Object()->simParameters->ldbStrategy == LDBSTRAT_NONE)
00505     return;
00506 
00507   iout << "LDB: ============= START OF LOAD BALANCING ============== " << CmiWallTimer() << "\n" << endi;
00508   DebugM(3, "Controller reached load balance barrier.\n");
00509   controllerReported = 1;
00510   controllerThread = c;
00511 
00512   CProxy_LdbCoordinator(thisgroup).barrier();
00513 
00514   CthSuspend();
00515 }

00494 {
00495   if (Node::Object()->simParameters->ldbStrategy == LDBSTRAT_NONE)
00496     return;
00497 
00498   sequencerThreads[pid] = seq;
00499   seq->suspend();
00500 }

00093 {
00094   theLbdb->RegisteringObjects(myHandle);
00095 }

00553 {
00554   // This method receives the migration from the framework,
00555   // unregisters it, and sends it to the destination PE
00556   const int id = m->handle.id.id[0];
00557 
00558   theLbdb->UnregisterObj(objHandles[id]);
00559   objHandles[id].id.id[0] = -1;
00560 
00561   CProxy_LdbCoordinator  ldbProxy(thisgroup);
00562   ldbProxy[m->to].ExpectMigrate(m);
00563 }

00633 {
00634   enum proxyHere { No, Yes };
00635   int numNodes = CkNumPes();
00636   proxyHere *proxyNodes = new proxyHere[numNodes];
00637   int nProxyNodes;
00638   int i;
00639 
00640   // Note all home patches.
00641   for ( i = 0; i < numNodes; ++i )
00642   {
00643     proxyNodes[i] = No;
00644   }
00645   nProxyNodes=0;
00646 
00647   // Check all two-away neighbors.
00648   // This is really just one-away neighbors, since 
00649   // two-away always returns zero: RKB
00650   PatchID neighbors[1 + PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
00651 
00652   int myNode = patchMap->node(id);
00653   neighbors[0] = id;
00654   int numNeighbors = 1 + patchMap->downstreamNeighbors(id,neighbors+1);
00655   for ( i = 0; i < numNeighbors; ++i )
00656   {
00657     const int proxyNode = patchMap->basenode(neighbors[i]);
00658     if (proxyNode != myNode)
00659       if (proxyNodes[proxyNode] == No)
00660       {
00661         proxyNodes[proxyNode] = Yes;
00662         neighborNodes[nProxyNodes] = proxyNode;
00663         nProxyNodes++;
00664       }
00665   }
00666 
00667   delete [] proxyNodes;
00668   return nProxyNodes;
00669 }

00583 {
00584   DebugM(3,"resume()\n");
00585 
00586   //  printLocalLdbReport();
00587 
00588   ldbCycleNum++;
00589   initialize(PatchMap::Object(),ComputeMap::Object(),1);
00590 }

00601 {
00602   DebugM(3,"resume2()\n");
00603 
00604 #if CONVERSE_VERSION_ELAN
00605   //  enableBlockingReceives();
00606 #endif
00607 
00608   awakenSequencers();
00609 }

00103 {
00104   theLbdb->DoneRegisteringObjects(myHandle);
00105 #if CMK_PERSISTENT_COMM
00106   if (takingLdbData) {
00107 //CmiPrintf("[%d] CmiDestoryPersistent\n", CkMyPe());
00108 
00109     HomePatchList *hpl = PatchMap::Object()->homePatchList();
00110     ResizeArrayIter<HomePatchElem> ai(*hpl);
00111     for (ai=ai.begin(); ai != ai.end(); ai++) {
00112       HomePatch *patch = (*ai).patch;
00113       patch->destoryPersistComm();
00114     }
00115   }
00116 #endif
00117   CProxy_LdbCoordinator cl(thisgroup);
00118   cl[0].nodeDone();
00119 }

00592                                              {
00593 
00594   DebugM(3,"resumeReady()\n");
00595   delete msg;
00596 
00597   CProxy_LdbCoordinator(thisgroup).resume2();
00598 }

00481 {
00482   CmiAssert(id >=0 && id < numComputes);
00483   theLbdb->ObjectStart(objHandles[id]);
00484 }

00057 {
00058    LdbCoordinator *ldbCoordinator = (LdbCoordinator *)LDOMUserData(handle.omhandle);
00059    ldbCoordinator->Migrate(handle,dest);
00060 }

00078 {
00079   CkPrintf("I'm supposed to query load\n");
00080 }

00083 {
00084 
00085 #if CONVERSE_VERSION_ELAN
00086     //disableBlockingReceives();
00087 #endif
00088 
00089   ((LdbCoordinator*)data)->ReceiveAtSync();
00090 }

00098 {
00099   ((LdbCoordinator*)data)->ResumeFromSync();
00100 }

00073 {
00074   CkPrintf("I'm supposed to set stats\n");
00075 }

00575                                          {
00576   DebugM(3,"updateComputesReady()\n");
00577 
00578   CProxy_LdbCoordinator(thisgroup).resume();
00579   CkStartQD(CkIndex_LdbCoordinator::resumeReady((CkQdMsg*)0),&thishandle);
00580 }

LdbCoordinator Class Reference

Public Member Functions

Static Public Member Functions

Public Attributes

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation


Public Member Functions
	LdbCoordinator ()
	~LdbCoordinator (void)
void	initialize (PatchMap pmap, ComputeMap cmap, int reinit=0)
void	createLoadBalancer ()
void	patchLoad (PatchID id, int nAtoms, int timestep)
void	startWork (ComputeID id, int timestep)
void	endWork (ComputeID id, int timestep)
void	rebalance (Sequencer *seq, PatchID id)
void	rebalance (Controller *seq)
void	nodeDone (void)
void	updateComputesReady ()
void	barrier (void)
void	resume (void)
void	resumeReady (CkQdMsg *msg)
void	resume2 (void)
int	getNumStepsToRun (void)
void	ReceiveAtSync (void)
void	Migrate (LDObjHandle handle, int dest)
void	RecvMigrate (LdbMigrateMsg *)
void	ProcessMigrate (LdbMigrateMsg *)
void	ExpectMigrate (LdbMigrateMsg *)
void	ResumeFromSync (void)
void	ExecuteMigrations (void)
void	awakenSequencers (void)
int	requiredProxies (PatchID id, int[])
void	printRequiredProxies (PatchID id, FILE *fp)
void	printLocalLdbReport (void)
Static Public Member Functions
LdbCoordinator *	Object ()
void	staticMigrateFn (LDObjHandle handle, int dest)
void	staticStatsFn (LDOMHandle h, int state)
void	staticQueryEstLoadFn (LDOMHandle h)
void	staticReceiveAtSync (void *data)
void	staticResumeFromSync (void *data)
Public Attributes
int	stepsPerLdbCycle
int	nLocalComputes
int	nLocalPatches
int	nPatchesReported
int	nPatchesExpected
int	nComputesReported
int	nComputesExpected
int	controllerReported
int	controllerExpected
int	nStatsMessagesReceived
int	nStatsMessagesExpected
ComputeMap *	computeMap
PatchMap *	patchMap
int *	patchNAtoms
int	nPatches
Controller *	controllerThread
Sequencer **	sequencerThreads
int	ldbCycleNum
int	numStepsToRun
int	firstLdbStep
int	totalStepsDone
int	takingLdbData
int	nodesDone
FILE *	ldbStatsFP
computeInfo *	computeArray
patchInfo *	patchArray
processorInfo *	processorArray
LBDatabase *	theLbdb
LDOMid	myOMid
LDOMHandle	myHandle
LDObjHandle *	objHandles
int	numComputes
int	nRegisteredObjs
LDBarrierClient	ldBarrierHandle
int	reg_all_objs
LDObjHandle *	patchHandles