NamdCentLB Class Reference

#include <NamdCentLB.h>

Inheritance diagram for NamdCentLB:

Public Member Functions
	NamdCentLB (const CkLBOptions &opt)
	NamdCentLB (CkMigrateMessage *)
CLBMigrateMsg *	Strategy (LDStats *stats)

Detailed Description

Definition at line 52 of file NamdCentLB.h.

Constructor & Destructor Documentation

NamdCentLB::NamdCentLB

(

const CkLBOptions &

opt

)

Definition at line 51 of file NamdCentLB.C.

                                            : CentralLB(opt)
{
  //  if (CkMyPe()==0)
  //   CkPrintf("[%d] NamdCentLB created\n",CkMyPe());
  processorArray = 0;
  patchArray = 0;
  computeArray = 0;
}

NamdCentLB::NamdCentLB

(

CkMigrateMessage *

msg

)

Migratable Object Constructor.

Definition at line 45 of file NamdCentLB.C.

                                           : CentralLB(msg) {
  processorArray = 0;
  patchArray = 0;
  computeArray = 0;
} 

Member Function Documentation

CLBMigrateMsg * NamdCentLB::Strategy

(

LDStats *

stats

)

Definition at line 88 of file NamdCentLB.C.

References averageLoad, processorInfo::backgroundLoad, cpuloads, endi(), computeInfo::handle, InfoRecord::Id, iINFO(), iout, SimParameters::LCPOOn, LdbIdField(), SimParameters::ldbRelativeGrainsize, LDBSTRAT_COMPREHENSIVE, LDBSTRAT_DEFAULT, LDBSTRAT_OLD, LDBSTRAT_REFINEONLY, SimParameters::ldbStrategy, InfoRecord::load, load, NAMD_die(), ComputeMap::numComputes(), ComputeMap::numPartitions(), PatchMap::numPatches(), numPatches, PatchMap::Object(), Node::Object(), ComputeMap::Object(), computeInfo::oldProcessor, computeInfo::processor, patchInfo::processor, ComputeMap::setNewNode(), ComputeMap::setNewNumPartitions(), Node::simParameters, and simParams.

{
  //  CkPrintf("LDB: All statistics received at %f, %f\n",
  //  CmiTimer(),CmiWallTimer());

  int numProcessors = stats->nprocs();
  int numPatches = PatchMap::Object()->numPatches();
  ComputeMap *computeMap = ComputeMap::Object();
  const int numComputes = computeMap->numComputes();
  const SimParameters* simParams = Node::Object()->simParameters;

  // these sizes should never change
  if ( ! processorArray ) processorArray = new processorInfo[numProcessors];
  if ( ! patchArray ) patchArray = new patchInfo[numPatches];
  if ( ! computeArray ) computeArray = new computeInfo[numComputes];

  int nMoveableComputes = buildData(stats);

#if LDB_DEBUG
#define DUMP_LDBDATA 1
#define LOAD_LDBDATA 1
#endif

#if DUMP_LDBDATA 
  dumpDataASCII("ldbd_before", numProcessors, numPatches, nMoveableComputes);
#elif LOAD_LDBDATA
  loadDataASCII("ldbd_before.5", numProcessors, numPatches, nMoveableComputes);
  // CkExit();
#endif

  double averageLoad = 0.;
  double avgCompute = 0.;
  if ( nMoveableComputes ) {
   int i;
   double total = 0.;
   double maxCompute = 0.;
   int maxi = 0;
   for (i=0; i<nMoveableComputes; i++) {
      double load = computeArray[i].load;
      total += load;
      if ( load > maxCompute ) { maxCompute = load;  maxi = i; }
   }
   avgCompute = total / nMoveableComputes;

    int P = stats->nprocs();
   int numPesAvailable = 0;
   for (i=0; i<P; i++) {
      if (processorArray[i].available) {
        ++numPesAvailable;
        total += processorArray[i].backgroundLoad;
      }
   }
   if (numPesAvailable == 0)
     NAMD_die("No processors available for load balancing!\n");

   averageLoad = total/numPesAvailable;
   CkPrintf("LDB: Largest compute %d load %f is %.1f%% of average load %f\n",
            LdbIdField(computeArray[maxi].handle.id, 0),
            maxCompute, 100. * maxCompute / averageLoad, averageLoad);
   CkPrintf("LDB: Average compute %f is %.1f%% of average load %f\n",
            avgCompute, 100. * avgCompute / averageLoad, averageLoad);
  }

  if ( step() == 1 ) {
    // compute splitting only
    // partitions are stored as char but mostly limited by
    // high load noise at low outer-loop iteration counts
    int maxParts = 10;
#if defined(NAMD_CUDA) || defined(NAMD_HIP)
//split LCPO compute very small, else CUDA compute is delayed
    if (simParams->LCPOOn) {
      maxParts = 20;
    }
#endif
    int totalAddedParts = 0;
    double maxCompute = averageLoad / 10.;
    if ( maxCompute < 2. * avgCompute ) maxCompute = 2. * avgCompute;
    if ( simParams->ldbRelativeGrainsize > 0. ) {
      maxCompute = averageLoad * simParams->ldbRelativeGrainsize;
    }
    CkPrintf("LDB: Partitioning computes with target load %f\n", maxCompute);
    double maxUnsplit = 0.;
    for (int i=0; i<nMoveableComputes; i++) {
      computeArray[i].processor = computeArray[i].oldProcessor;
      const int cid = LdbIdField(computeArray[i].handle.id, 0);
      const double load = computeArray[i].load;
      if ( computeMap->numPartitions(cid) == 0 ) {
        if ( load > maxUnsplit ) maxUnsplit = load;
        continue;
      }
      int nparts = (int) ceil(load / maxCompute);
      if ( nparts > maxParts ) nparts = maxParts;
      if ( nparts < 1 ) nparts = 1;
      if ( 0 && nparts > 1 ) {
        CkPrintf("LDB: Partitioning compute %d with load %f by %d\n",
                  cid, load, nparts);
      }
      computeMap->setNewNumPartitions(cid,nparts);
      totalAddedParts += nparts - 1;
    }
    CkPrintf("LDB: Increased migratable compute count from %d to %d\n",
              nMoveableComputes,nMoveableComputes+totalAddedParts);
    CkPrintf("LDB: Largest unpartitionable compute is %f\n", maxUnsplit);
  } else if (simParams->ldbStrategy == LDBSTRAT_DEFAULT) { // default
    if (step() < 4)
      TorusLB(computeArray, patchArray, processorArray,
                  nMoveableComputes, numPatches, numProcessors);
    else
      RefineTorusLB(computeArray, patchArray, processorArray,
                  nMoveableComputes, numPatches, numProcessors, 1);
  } else if (simParams->ldbStrategy == LDBSTRAT_COMPREHENSIVE) {
    TorusLB(computeArray, patchArray, processorArray,
                  nMoveableComputes, numPatches, numProcessors);
  } else if (simParams->ldbStrategy == LDBSTRAT_REFINEONLY) {
    RefineTorusLB(computeArray, patchArray, processorArray,
                  nMoveableComputes, numPatches, numProcessors, 1);
  } else if (simParams->ldbStrategy == LDBSTRAT_OLD) {
    if (step() < 4)
      Alg7(computeArray, patchArray, processorArray,
                  nMoveableComputes, numPatches, numProcessors);
    else
      RefineOnly(computeArray, patchArray, processorArray, 
                  nMoveableComputes, numPatches, numProcessors);
  }

#if LDB_DEBUG && USE_TOPOMAP
  TopoManager tmgr;
  int pe1, pe2, pe3, hops=0;
  /* This is double counting the hops
  for(int i=0; i<nMoveableComputes; i++)
  {
    pe1 = computeArray[i].processor;
    pe2 = patchArray[computeArray[i].patch1].processor;
    pe3 = patchArray[computeArray[i].patch2].processor;
    hops += tmgr.getHopsBetweenRanks(pe1, pe2);
    if(computeArray[i].patch1 != computeArray[i].patch2)
      hops += tmgr.getHopsBetweenRanks(pe1, pe3);  
  }*/
  for (int i=0; i<numPatches; i++)  {
    //int num = patchArray[i].proxiesOn.numElements();
    pe1 = patchArray[i].processor;
    Iterator nextProc;
    processorInfo *p = (processorInfo *)patchArray[i].proxiesOn.iterator((Iterator *)&nextProc);
    while (p) {
      pe2 = p->Id;
      hops += tmgr.getHopsBetweenRanks(pe1, pe2);
      p = (processorInfo *)patchArray[i].proxiesOn.next((Iterator*)&nextProc);
    }
  }
  CkPrintf("Load Balancing: Number of Hops: %d\n", hops);
#endif

#if DUMP_LDBDATA
  dumpDataASCII("ldbd_after", numProcessors, numPatches, nMoveableComputes);
#elif LOAD_LDBDATA
  dumpDataASCII("ldbd_after.5", numProcessors, numPatches, nMoveableComputes);
  // loadDataASCII("ldbd_after", numProcessors, numPatches, nMoveableComputes);
  // CkExit();
#endif

  // For error checking:
  // Count up computes, to see if somebody doesn't have any computes
  int i;
#if 0
  int* computeCount = new int[numProcessors];
  for(i=0; i<numProcessors; i++)
    computeCount[i]=0;
  for(i=0; i<nMoveableComputes; i++)
    computeCount[computeArray[i].processor]++;
  for(i=0; i<numProcessors; i++) {
    if (computeCount[i]==0)
      iout << iINFO <<"Warning: Processor " << i 
           << " has NO moveable computes.\n" << endi;
  }
  delete [] computeCount;
#endif
  
  std::vector<MigrateInfo *> migrateInfo;
  for(i=0;i<nMoveableComputes;i++) {
    if (computeArray[i].processor != computeArray[i].oldProcessor) {
      //      CkPrintf("[%d] Obj %d migrating from %d to %d\n",
      //               CkMyPe(),computeArray[i].handle.id.id[0],
      //               computeArray[i].processor,computeArray[i].oldProcessor);
      MigrateInfo *migrateMe = new MigrateInfo;
      migrateMe->obj = computeArray[i].handle;
      migrateMe->from_pe = computeArray[i].oldProcessor;
      migrateMe->to_pe = computeArray[i].processor;
      migrateInfo.push_back(migrateMe);

      // sneak in updates to ComputeMap
      computeMap->setNewNode(LdbIdField(computeArray[i].handle.id, 0),
                                computeArray[i].processor);
    }
  }
  
  const int migrate_count=migrateInfo.size();
  // CkPrintf("NamdCentLB migrating %d elements\n",migrate_count);
  CLBMigrateMsg* msg = new(migrate_count,CkNumPes(),CkNumPes(),0) CLBMigrateMsg;

  msg->n_moves = migrate_count;
  for(i=0; i < migrate_count; i++) {
    MigrateInfo* item = migrateInfo[i];
    msg->moves[i] = *item;
    delete item;
    migrateInfo[i] = nullptr;
  }

  for (i=0; i<numProcessors; i++) {
    cpuloads[i] = processorArray[i].load;
  }

  delete [] processorArray;
  delete [] patchArray;
  delete [] computeArray;

  processorArray = NULL;
  patchArray = NULL;
  computeArray = NULL;
  
  return msg;
};

---

The documentation for this class was generated from the following files:

• NamdCentLB.h
• NamdCentLB.C