NamdNborLB.C

Go to the documentation of this file.
00001 
00002 #if !defined(WIN32) || defined(__CYGWIN__)
00003 #include <unistd.h>
00004 #endif
00005 #include <fcntl.h>
00006 
00007 #include "InfoStream.h"
00008 #include "NamdNborLB.h"
00009 #include "NamdNborLB.def.h"
00010 #include "Node.h"
00011 #include "PatchMap.h"
00012 #include "ComputeMap.h"
00013 #include "LdbCoordinator.h"
00014 
00015 void CreateNamdNborLB()
00016 {
00017   // CkPrintf("[%d] creating NamdNborLB %d\n",CkMyPe(),loadbalancer);
00018   CProxy_NamdNborLB::ckNew();
00019   // CkPrintf("[%d] created NamdNborLB %d\n",CkMyPe(),loadbalancer);
00020 }
00021 
00022 NamdNborLB::NamdNborLB(): NeighborLB(CkLBOptions(-1))
00023 {
00024   //  if (CkMyPe()==0)
00025   //   CkPrintf("[%d] NamdNborLB created\n",CkMyPe());
00026   processorArray = 0;
00027   patchArray = 0;
00028   computeArray = 0;
00029   act = 0;
00030   ldbNum = 0;
00031 }
00032 
00033 /*
00034 NamdNborLB::~NamdNborLB()
00035 {
00036   delete [] processorArray;
00037   delete [] patchArray;
00038   delete [] computeArray;
00039 }
00040 */
00041 
00042 int NamdNborLB::max_neighbors() {
00043   return 4;
00044 }
00045 
00046 int NamdNborLB::num_neighbors() {
00047   return numNbors;
00048 }
00049 
00050 void NamdNborLB::neighbors(int* _n) {
00051 #if 0
00052     const int me = CkMyPe();
00053     const int npe = CkNumPes();
00054     if (npe > 1)
00055       _n[0] = (me + npe - 1) % npe;
00056     if (npe > 2)
00057       _n[1] = (me + 1) % npe;
00058 
00059     int bigstep = (npe - 1) / 3 + 1;
00060     if (bigstep == 1) bigstep++;
00061 
00062     if (npe > 3)
00063       _n[2] = (me + bigstep) % npe;
00064     if (npe > 4)
00065       _n[3] = (me + npe - bigstep) % npe;
00066 #else   // 2D mesh or Torus mesh
00067 #define SEQ(x, y) ((x)*yDim + (y))
00068 #define WRAP   0
00069     numNbors = 0;
00070     int yDim = (int)sqrt((double)CkNumPes());
00071     int xDim = CkNumPes() / yDim;
00072     if (CkNumPes() % yDim) xDim++;
00073     int x = CkMyPe()/yDim;
00074     int y = CkMyPe()%yDim;
00075     int x1, y1, s;
00076     // CmiPrintf("[%d]info: %d %d %d %d\n", CkMyPe(), xDim, yDim, x,y);
00077 
00078     x1=x; y1 = y-1;
00079 #if WRAP
00080     if (y1==-1) y1=yDim-1;
00081     if (SEQ(x1, y1) >= CkNumPes()) s = CkNumPes()-1;
00082     else s = SEQ(x1, y1);
00083     if (s != CkMyPe()) _n[numNbors++] = s;
00084 #else
00085     if (y1 != -1)  _n[numNbors++] = SEQ(x1, y1);
00086 #endif
00087 
00088     x1=x; y1=y+1;
00089 #if WRAP
00090     if (y1 == yDim || SEQ(x1,y1) >= CkNumPes()) y1=0;
00091     s = SEQ(x1, y1);
00092     if (s != _n[numNbors-1] && s != CkMyPe()) _n[numNbors++] = s;
00093 #else
00094     if (y1 == yDim || SEQ(x1,y1) >= CkNumPes()) ;
00095     else _n[numNbors++] = SEQ(x1, y1);
00096 #endif
00097 
00098     y1=y; x1=x-1;
00099 #if WRAP
00100     if (x1==-1) x1=xDim-1;
00101     if (SEQ(x1, y1) >= CkNumPes()) x1--;
00102     s = SEQ(x1, y1);
00103     if (s != CkMyPe()) _n[numNbors++] = s;
00104 #else
00105     if (x1!=-1) _n[numNbors++] = SEQ(x1, y1);
00106 #endif
00107 
00108     y1=y; x1=x+1;
00109 #if WRAP
00110     if (x1==xDim || SEQ(x1,y1) >= CkNumPes()) x1=0;
00111     s = SEQ(x1, y1);
00112     if (s != _n[numNbors-1] && s != CkMyPe()) _n[numNbors++] = s;
00113 #else
00114     if (x1==xDim || SEQ(x1,y1) >= CkNumPes()) ;
00115     else _n[numNbors++] = SEQ(x1,y1);
00116 #endif
00117     // CmiPrintf("[%d] %d neighbors: %d %d %d %d\n", CkMyPe(), numNbors, _n[0], _n[1], _n[2], _n[3]);
00118     act = (x+y)%2;
00119 #endif
00120 
00121 };
00122 
00123 bool NamdNborLB::QueryBalanceNow(int _step)
00124 {
00125   // CkPrintf("[%d] QueryBalanceNow on step %d: %d\n",CkMyPe(),_step, LdbCoordinator::Object()->takingLdbData);
00126   if ( LdbCoordinator::Object()->takingLdbData ) {
00127     return true;
00128   } else {
00129     return false;
00130   }
00131 }
00132 
00133 bool NamdNborLB::QueryMigrateStep(int _step)
00134 {
00135   // CmiPrintf("[%d] QueryMigrateStep %d %d.\n", CkMyPe(), _step, act);
00136   return (act+ldbNum)%2 == 0;
00137 }
00138 
00139 NLBMigrateMsg* NamdNborLB::Strategy(NborBaseLB::LDStats* stats, int count)
00140 {
00141   //  CkPrintf("LDB:[%d] All statistics received at %f, %f\n",
00142   //  CkMyPe(), CmiTimer(),CmiWallTimer());
00143   int i,j;
00144   int mype = CkMyPe();
00145 
00146   ldbNum ++;
00147 
00148   const int numProcessors = CkNumPes();
00149   const int numPatches = PatchMap::Object()->numPatches();
00150   const int numComputes = ComputeMap::Object()->numComputes();
00151   const SimParameters* simParams = Node::Object()->simParameters;
00152 
00153   int nMoveableComputes = 0;
00154   for (i=0; i < count+1; i++) {
00155     LDStats &thisLDStats = ((i==count)?myStats:stats[i]);
00156     for (j=0; j < thisLDStats.n_objs; j++) {
00157       const LDObjData &this_obj = thisLDStats.objData[j];
00158       if (this_obj.omID().id.idx != 1) continue;
00159       if (this_obj.id().id[1] == -2) continue;
00160       if (this_obj.migratable)  nMoveableComputes++;
00161     }
00162   }
00163   // CmiPrintf("%d nMoveableComputes: %d\n", CkMyPe(), nMoveableComputes);
00164 
00165   // these sizes should never change
00166   processorArray = new processorInfo[numProcessors];
00167   patchArray = new patchInfo[numPatches];
00168 //  if ( ! computeArray ) computeArray = new computeInfo[nMoveableComputes];
00169   computeArray = new computeInfo[nMoveableComputes];
00170 
00171   nMoveableComputes = buildData(stats,count);
00172 
00173   //CmiPrintf("AlgNbor begin on %d\n", CkMyPe());
00174   AlgNbor(mype, computeArray,patchArray,processorArray,
00175                         nMoveableComputes, numPatches, numProcessors, count);
00176   //CmiPrintf("AlgNbor end on %d\n", CkMyPe());
00177 
00178   CkVec<MigrateInfo *> migrateInfo;
00179   for(i=0;i<nMoveableComputes;i++) {
00180     if (computeArray[i].oldProcessor == mype)
00181     if (computeArray[i].processor != computeArray[i].oldProcessor) {
00182       // CkPrintf("[%d] Obj %d migrating from %d to %d\n",
00183       //          CkMyPe(),computeArray[i].handle.id.id[0],
00184       //       computeArray[i].processor,computeArray[i].oldProcessor);
00185       MigrateInfo *migrateMe = new MigrateInfo;
00186       migrateMe->obj = computeArray[i].handle;
00187       migrateMe->from_pe = computeArray[i].oldProcessor;
00188       migrateMe->to_pe = computeArray[i].processor;
00189       migrateInfo.insertAtEnd(migrateMe);
00190     }
00191   }
00192   
00193   int migrate_count=migrateInfo.length();
00194   CkPrintf("NamdNborLB [%d] migrating %d elements\n", CkMyPe(), migrate_count);
00195   NLBMigrateMsg* msg = new(migrate_count,CkNumPes(),CkNumPes(),0) NLBMigrateMsg;
00196   msg->n_moves = migrate_count;
00197   for(i=0; i < migrate_count; i++) {
00198     MigrateInfo* item = migrateInfo[i];
00199     msg->moves[i] = *item;
00200     delete item;
00201     migrateInfo[i] = 0;
00202   }
00203 
00204   delete [] patchArray;
00205   delete [] computeArray;
00206   /*
00207   for(i=0; i<numProcessors; i++)
00208       delete [] processorArray[i].proxyUsage;
00209   */
00210   delete [] processorArray;
00211 
00212   return msg;
00213 };
00214 
00215 
00216 int NamdNborLB::buildData(NborBaseLB::LDStats* stats, int count)
00217 {
00218   PatchMap* patchMap = PatchMap::Object();
00219   ComputeMap* computeMap = ComputeMap::Object();
00220   double bg_weight = 0.7;
00221 
00222   int i;
00223   for (i=0; i<CkNumPes(); i++) {
00224     processorArray[i].load = 0.0;
00225     processorArray[i].backgroundLoad = 0.0;
00226     processorArray[i].available = false;
00227     if (i == CkMyPe()) {
00228       processorArray[i].Id = i;
00229       processorArray[i].available = myStats.move;
00230     if (patchMap->numPatches() > 0)
00231       processorArray[i].backgroundLoad = myStats.bg_walltime*bg_weight;
00232     else 
00233       processorArray[i].backgroundLoad = myStats.bg_walltime;
00234       continue;
00235     }
00236     int peslot = NeighborIndex(i);
00237     if (peslot != -1) {
00238     processorArray[i].Id = i;
00239       processorArray[i].available = stats[peslot].move;
00240     if (patchMap->numPatches() > 0)
00241       processorArray[i].backgroundLoad = bg_weight * stats[peslot].bg_walltime;
00242     else 
00243       processorArray[i].backgroundLoad = stats[peslot].bg_walltime;
00244     }
00245     else 
00246       processorArray[i].Id = -2;
00247   }
00248 
00249   int nMoveableComputes=0;
00250   int nProxies = 0;             // total number of estimated proxies
00251   for (i=0; i<patchMap->numPatches(); i++) {
00252         patchArray[i].Id = i;
00253         patchArray[i].numAtoms = 0;
00254         patchArray[i].processor = patchMap->node(i);
00255         int neighborNodes[PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
00256         const int numProxies = requiredProxies(i,neighborNodes);
00257         nProxies += numProxies;
00258 
00259         for (int k=0; k<numProxies; k++) {
00260           if (NeighborIndex(neighborNodes[k]) != -1) {
00261             processorArray[neighborNodes[k]].proxies.unchecked_insert(&patchArray[i]);
00262             patchArray[i].proxiesOn.unchecked_insert(&processorArray[neighborNodes[k]]);
00263           }
00264         }
00265   }
00266   for (i=0; i < count+1; i++) {
00267     int j;
00268     LDStats &thisLDStats = ((i==count)?myStats:stats[i]);
00269     for (j=0; j < thisLDStats.n_objs; j++) {
00270       const LDObjData &this_obj = thisLDStats.objData[j];
00271       // filter out non-NAMD managed objects (like PME array)
00272       if (this_obj.omID().id.idx != 1) continue;
00273       if (this_obj.id().id[1] == -2) { // Its a patch
00274 /*
00275         const int pid = this_obj.id.id[0];
00276         int neighborNodes[PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
00277 
00278         patchArray[pid].Id = pid;
00279         patchArray[pid].numAtoms = 0;
00280         patchArray[pid].processor = i;
00281         const int numProxies = requiredProxies(pid,neighborNodes);
00282         nProxies += numProxies;
00283 
00284         for (int k=0; k<numProxies; k++) {
00285           processorArray[neighborNodes[k]].proxies.unchecked_insert(&patchArray[pid]);
00286           patchArray[pid].proxiesOn.unchecked_insert(&processorArray[neighborNodes[k]]);
00287         }
00288 */
00289       } else if (this_obj.migratable) { // Its a compute
00290         const int cid = this_obj.id().id[0];
00291         const int p0 = computeMap->pid(cid,0);
00292 
00293         // For self-interactions, just return the same pid twice
00294         int p1;
00295         if (computeMap->numPids(cid) > 1)
00296           p1 = computeMap->pid(cid,1);
00297         else p1 = p0;
00298         computeArray[nMoveableComputes].Id = cid;
00299         computeArray[nMoveableComputes].oldProcessor = thisLDStats.from_pe;
00300         computeArray[nMoveableComputes].processor = -1;
00301         computeArray[nMoveableComputes].patch1 = p0;
00302         computeArray[nMoveableComputes].patch2 = p1;
00303         computeArray[nMoveableComputes].handle = this_obj.handle;
00304         computeArray[nMoveableComputes].load = this_obj.wallTime;
00305         nMoveableComputes++;
00306       }
00307     }
00308   }
00309   return nMoveableComputes;
00310 }
00311 
00312 // Figure out which proxies we will definitely create on other
00313 // nodes, without regard for non-bonded computes.  This code is swiped
00314 // from ProxyMgr, and changes there probable need to be propagated here.
00315 
00316 int NamdNborLB::requiredProxies(PatchID id, int neighborNodes[])
00317 {
00318   enum proxyHere { No, Yes };
00319   int numNodes = CkNumPes();
00320   proxyHere *proxyNodes = new proxyHere[numNodes];
00321   int nProxyNodes;
00322   int i;
00323 
00324   // Note all home patches.
00325   for ( i = 0; i < numNodes; ++i )
00326   {
00327     proxyNodes[i] = No;
00328   }
00329   nProxyNodes=0;
00330 
00331   // Check all two-away neighbors.
00332   // This is really just one-away neighbors, since 
00333   // two-away always returns zero: RKB
00334   PatchID neighbors[1 + PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
00335 
00336   PatchMap* patchMap = PatchMap::Object();
00337 
00338   int myNode = patchMap->node(id);
00339   neighbors[0] = id;
00340   int numNeighbors = 1 + patchMap->downstreamNeighbors(id,neighbors+1);
00341   for ( i = 0; i < numNeighbors; ++i )
00342   {
00343     const int proxyNode = patchMap->basenode(neighbors[i]);
00344     if (proxyNode != myNode)
00345       if (proxyNodes[proxyNode] == No)
00346       {
00347         proxyNodes[proxyNode] = Yes;
00348         neighborNodes[nProxyNodes] = proxyNode;
00349         nProxyNodes++;
00350       }
00351   }
00352 
00353   delete [] proxyNodes;
00354   return nProxyNodes;
00355 }
00356 

Generated on Fri Sep 22 01:17:13 2017 for NAMD by  doxygen 1.4.7