ProxyMgr.h

Go to the documentation of this file.
00001 
00007 #ifndef PROXYMGR_H
00008 #define PROXYMGR_H
00009 
00010 
00011 #include "charm++.h"
00012 #include "envelope.h"
00013 
00014 #include "main.h"
00015 #include "NamdTypes.h"
00016 #include "PatchTypes.h"
00017 #include "UniqueSet.h"
00018 #include "UniqueSetIter.h"
00019 #include "ProcessorPrivate.h"
00020 #include "ProxyMgr.decl.h"
00021 
00022 extern int proxySendSpanning, proxyRecvSpanning;
00023 extern int proxySpanDim;
00024 extern int inNodeProxySpanDim;
00025 
00026 #if CMK_PERSISTENT_COMM
00027 #define USE_PERSISTENT_TREE                  0
00028 #endif
00029 
00030 class ProxyGBISP1ResultMsg: public CMessage_ProxyGBISP1ResultMsg {
00031   public:
00032     int destPe;
00033     int origPe;
00034     PatchID patch;
00035     GBReal *psiSum;
00036     int psiSumLen;// = numAtoms
00037 };
00038 class ProxyGBISP2DataMsg: public CMessage_ProxyGBISP2DataMsg {
00039 public:
00040     int origPe;
00041     int destPe;
00042     PatchID patch;
00043     Real *bornRad;//numAtoms
00044     int bornRadLen;
00045   };
00046 class ProxyGBISP2ResultMsg: public CMessage_ProxyGBISP2ResultMsg {
00047 public:
00048     int destPe;
00049     int origPe;
00050     PatchID patch;
00051     GBReal *dEdaSum;
00052     int dEdaSumLen;//numAtoms
00053   };
00054 class ProxyGBISP3DataMsg: public CMessage_ProxyGBISP3DataMsg {
00055 public:
00056     int origPe;
00057     int destPe;
00058     PatchID patch;
00059     Real *dHdrPrefix;//numAtoms
00060     int dHdrPrefixLen;
00061   };
00062 
00063 class RegisterProxyMsg : public CMessage_RegisterProxyMsg {
00064 public:
00065   NodeID node;
00066   PatchID patch;
00067 };
00068 
00069 class UnregisterProxyMsg : public CMessage_UnregisterProxyMsg {
00070 public:
00071   NodeID node;
00072   PatchID patch;
00073 };
00074 
00075 //1. This class represents for both msg types: one that
00076 //is originally known as ProxyAllMsg which is sent
00077 //at the step where atoms migrate; and the other is
00078 //sent during the steps between two migrations.
00079 //2. In the case of memory optimized version, the scenario
00080 //becomes tricky as load balancer will move compute objects
00081 //around so that new ProxyPatches will be created where
00082 //the CompAtomExt list information is not available. If
00083 //the step immediately after the load balancing is a normal
00084 //step, then the CompAtomExt list info has to be resent by
00085 //the HomePatch. Because of the current Proxy msg communication
00086 //scheme where msg is sent to ProxyMgr first, and then retransmitted
00087 //to ProxyPatches, there's overhead when we want to resend CompAtomExt
00088 //list as not all the ProxyPatches that are managed by ProxyMgr are
00089 //newly created ProxyPatches. 
00090 //--Chao Mei
00091 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
00092 class ProxyDataMsg_not_32_byte : public CMessage_ProxyDataMsg {
00093 #else
00094 class ProxyDataMsg : public CMessage_ProxyDataMsg {
00095 #endif
00096 public:
00097   PatchID patch;
00098   Flags flags;
00099 
00100   int plLen;
00101 
00102   CompAtom *positionList;
00103   int avgPlLen;
00104   CompAtom *avgPositionList;
00105   // BEGIN LA
00106   int vlLen;
00107   CompAtom *velocityList;
00108   // END LA
00109 
00110   Real *intRadList;// gbis atom intrinsic radii
00111 
00112   int *lcpoTypeList;// LCPO atom type
00113 
00114   //1. The following field will be only
00115   //useful for memory optimized version.
00116   //2. In normal case, adding this field only
00117   //increases the msg length by 4 bytes which
00118   //can be ignored considering the current fast
00119   //communication network
00120   //--Chao Mei
00121   int plExtLen;
00122   CompAtomExt *positionExtList;
00123   CudaAtom *cudaAtomList;
00124 
00125 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
00126   //In smp layer, the couter for msg creation and process of communication
00127   //thread is not included in the quiescence detection process. In addition,
00128   //the immediate messages from other nodes are executed on the communication
00129   //thread. If inside the process of immediate messages, some normal Charm++
00130   //messages sent out which will be processed on worker threads. Then QD will
00131   //be a problem that the process of the normal messages sent from communication
00132   //thread is recorded, but the creation of such messages (although recorded
00133   //in the comm thread) is virtually not recorded, i.e., not visible the 
00134   //QD process. So we need to artificially increase the QD counter to 
00135   //compensate for aforementioned msg creation loss.
00136   //The idea is to use the following variable to indicate the normal message
00137   //is sent from the communication thread inside a processing of immediate
00138   //message. If the variable is set, then we should increase the QD counter.
00139   //Chao Mei
00140   char isFromImmMsgCall; //hack for imm msg with QD in SMP 
00141 #endif
00142 
00143   // DMK - Atom Separation (water vs. non-water)
00144   #if NAMD_SeparateWaters != 0
00145     int numWaterAtoms;  // Number of atoms in positionList (from start)
00146                         //   that are part of water hydrogen groups.
00147   #endif
00148 
00149 };
00150 
00151 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
00152 class ProxyDataMsg : public ProxyDataMsg_not_32_byte {
00153   // Adding padding bytes to make sure that positionList is
00154   // 32-byte aligned which usually gives better cache performance.
00155   char padding[(32-(sizeof(envelope)+sizeof(ProxyDataMsg_not_32_byte))%32)%32];
00156 };
00157 class assert_ProxyDataMsg {
00158   char assert_sizeof_envelope_is_multiple_of_ptr[(sizeof(envelope)%sizeof(void*))?-1:1];
00159   char assert_sizeof_ProxyDataMsg_is_multiple_of_32[((sizeof(envelope)+sizeof(ProxyDataMsg))%32)?-1:1];
00160 };
00161 #endif
00162 
00163 
00164 class ProxyResultMsg : public CMessage_ProxyResultMsg {
00165 public:
00166   NodeID node;
00167   PatchID patch;
00168   ForceList *forceList[Results::maxNumForces];
00169   static void* pack(ProxyResultMsg *msg);
00170   static ProxyResultMsg* unpack(void *ptr);
00171 private:
00172   ForceList forceListInternal[Results::maxNumForces];
00173 };
00174 
00175 class ProxyResultVarsizeMsg: public CMessage_ProxyResultVarsizeMsg{
00176 public:
00177     NodeID node;
00178     PatchID patch;
00179     int flLen[Results::maxNumForces];   
00180 
00181     Force *forceArr;
00182     //Indicate the position of the force list that has zero value
00183     //which is not recorded in the above force array.
00184     char *isZero;
00185 
00186     //add padding bytes to make sure the beginning 
00187     //of force arrays is 8-byte aligned as it is originally.
00188     //Therefore, we have to put the forceArr field as
00189     //the first variable of varsize array type
00190     char padding[(8-(sizeof(envelope)+sizeof(NodeID)+sizeof(PatchID)+sizeof(int)*Results::maxNumForces+2*sizeof(void *))%8)%8];   
00191 
00192     //The length of "fls" is Results::maxNumForces
00193     static ProxyResultVarsizeMsg *getANewMsg(NodeID nid, PatchID pid, int prioSize, ForceList *fls); 
00194 };
00195 
00196 class assert_ProxyResultVarsizeMsg {
00197   char assert_sizeof_envelope_is_multiple_of_ptr[(sizeof(envelope)%sizeof(void*))?-1:1];
00198   char assert_sizeof_ProxyResultVarsizeMsg_is_multiple_of_8[((sizeof(envelope)+sizeof(ProxyResultVarsizeMsg))%8)?-1:1];
00199 };
00200 
00201 class ProxyNodeAwareSpanningTreeMsg: public CMessage_ProxyNodeAwareSpanningTreeMsg{
00202 public:
00203     PatchID patch;
00204     NodeID procID;
00205     int numNodesWithProxies;
00206     int *numPesOfNode;
00207     int *allPes;
00208 
00209     static ProxyNodeAwareSpanningTreeMsg *getANewMsg(PatchID pid, NodeID nid, proxyTreeNode *tree, int size);
00210 
00211     //For debug
00212     void printOut(char *tag);
00213 };
00214 
00215 class ProxyCombinedResultRawMsg : public CMessage_ProxyCombinedResultRawMsg {
00216 public:
00217         int nodeSize;
00218         NodeID *nodes;
00219 
00220         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00221         //since this msg may be processed by comm thread in the smp mode,
00222         //this variable helps comm thread to find which proc will actually process it.
00223         NodeID destPe;
00224         #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
00225         //Mainly for QD in the presence of the optimization of using immediate
00226         //message. Refer to the explanation from ProxyDataMsg for the same 
00227         //variable. --Chao Mei
00228         char isFromImmMsgCall;
00229         #endif
00230         #endif
00231         PatchID patch;
00232 
00233         int flLen[Results::maxNumForces];
00234         char *isForceNonZero;
00235         //The beginning address of this variable should be 8-byte aligned!!! -Chao Mei
00236         Force *forceArr;
00237 };
00238 
00239 class ProxyCombinedResultMsg : public CMessage_ProxyCombinedResultMsg {
00240 public:
00241   #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00242   //since this msg may be processed by comm thread in the smp mode,
00243   //this variable helps comm thread to find which proc will actually process it.
00244   NodeID destPe;
00245   #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
00246   //Mainly for QD in the presence of the optimization of using immediate
00247   //message. Refer to the explanation from ProxyDataMsg for the same 
00248   //variable. --Chao Mei
00249   char isFromImmMsgCall;
00250   #endif
00251   #endif
00252   PatchID patch;
00253   NodeIDList nodes;
00254   ForceList *forceList[Results::maxNumForces];
00255   static ProxyCombinedResultRawMsg* toRaw(ProxyCombinedResultMsg *msg);
00256   static ProxyCombinedResultMsg* fromRaw(ProxyCombinedResultRawMsg *msg);
00257 private:
00258   ForceList forceListInternal[Results::maxNumForces];
00259 };
00260 
00261 class ProxySpanningTreeMsg : public CMessage_ProxySpanningTreeMsg {
00262 public:
00263   PatchID patch;
00264   NodeID  node;
00265   NodeIDList tree;
00266   static void* pack(ProxySpanningTreeMsg *msg);
00267   static ProxySpanningTreeMsg* unpack(void *ptr);
00268 };
00269 
00270 class ProxyPatch;
00271 class PatchMap;
00272 
00273 struct ProxyElem {
00274   ProxyElem() : proxyPatch(0) { };
00275   ProxyElem(PatchID pid) : patchID(pid), proxyPatch(0) { };
00276   ProxyElem(PatchID pid, ProxyPatch *p) : patchID(pid), proxyPatch(p) { };
00277 
00278   int hash() const { return patchID; }
00279   int operator==(const ProxyElem & pe) const { return patchID == pe.patchID; }
00280 
00281   PatchID patchID;
00282   ProxyPatch *proxyPatch;
00283 };
00284 
00285 typedef UniqueSet<ProxyElem> ProxySet;
00286 typedef UniqueSetIter<ProxyElem> ProxySetIter;
00287 
00288 class ProxyTree {       // keep track of the spanning trees
00289   public:
00290     int proxyMsgCount;
00291     NodeIDList *proxylist;
00292 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00293     //a node-aware spanning tree array, each element of which
00294     //is a spanning tree for all proxies of a patch
00295     proxyTreeNodeList *naTrees;
00296 #else
00297     NodeIDList *trees;
00298     int *sizes;
00299 #endif
00300     
00301   public:
00302     ProxyTree() {
00303       proxyMsgCount = 0;
00304       proxylist = NULL;
00305 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00306       naTrees = NULL;
00307 #else
00308       trees = NULL;
00309       sizes = NULL;
00310 #endif      
00311     }
00312     ~ProxyTree() {
00313     }
00314 };
00315 
00316 class ProxyMgr : public CBase_ProxyMgr
00317 {
00318 public:
00319   ProxyMgr();
00320   ~ProxyMgr();
00321 
00322   void removeProxies(void);
00323   void removeUnusedProxies(void);
00324   void createProxies(void);
00325 
00326   void createProxy(PatchID pid);
00327   void removeProxy(PatchID pid);
00328 
00329   void registerProxy(PatchID pid);
00330   void recvRegisterProxy(RegisterProxyMsg *);
00331 
00332   void unregisterProxy(PatchID pid);
00333   void recvUnregisterProxy(UnregisterProxyMsg *);
00334 
00335   void setSendSpanning();
00336   int  getSendSpanning();
00337 
00338   void setRecvSpanning();
00339   int  getRecvSpanning();
00340 
00341   void setProxyTreeBranchFactor(int dim);
00342 
00343   void buildProxySpanningTree();
00344   void sendSpanningTrees();
00345   void sendSpanningTreeToHomePatch(int pid, int *tree, int n);
00346   void recvSpanningTreeOnHomePatch(int pid, int *tree, int n);
00347   void sendSpanningTree(ProxySpanningTreeMsg *);
00348   void recvSpanningTree(ProxySpanningTreeMsg *);
00349 
00350   void sendNodeAwareSpanningTreeToHomePatch(int pid, proxyTreeNode *tree, int n);
00351   void recvNodeAwareSpanningTreeOnHomePatch(ProxyNodeAwareSpanningTreeMsg *msg);
00352   void sendNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
00353   void recvNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
00354   //set the proxy patch's parent field
00355   void recvNodeAwareSTParent(int patch, int parent);
00356 
00357   void buildProxySpanningTree2();               // centralized version
00358   void sendProxies(int pid, int *list, int n);
00359   void recvProxies(int pid, int *list, int n);
00360   void recvPatchProxyInfo(PatchProxyListMsg *msg);
00361 
00362 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00363   void buildNodeAwareSpanningTree0();
00364   static void buildSinglePatchNodeAwareSpanningTree(PatchID pid, NodeIDList &proxyList, 
00365                                                     proxyTreeNodeList &ptnTree);
00366 #else
00367   void buildSpanningTree0();
00368 #endif
00369 
00370   void sendResults(ProxyResultVarsizeMsg *);
00371   void recvResults(ProxyResultVarsizeMsg *);
00372   void sendResults(ProxyResultMsg *);
00373   void recvResults(ProxyResultMsg *);
00374   void sendResults(ProxyCombinedResultMsg *);
00375 
00376   void sendResult(ProxyGBISP1ResultMsg *);//psiSum
00377   void recvResult(ProxyGBISP1ResultMsg *);
00378   void recvData(   ProxyGBISP2DataMsg *);
00379   void sendResult(ProxyGBISP2ResultMsg *);//dEdaSum
00380   void recvResult(ProxyGBISP2ResultMsg *);
00381   void recvData(   ProxyGBISP3DataMsg *);
00382 
00383   void recvResults(ProxyCombinedResultRawMsg *);
00384   void recvImmediateResults(ProxyCombinedResultRawMsg *);
00385 
00386   void sendProxyData(ProxyDataMsg *, int, int*);
00387   void recvImmediateProxyData(ProxyDataMsg *);
00388   void recvProxyData(ProxyDataMsg *);
00389 
00390   void sendProxyAll(ProxyDataMsg *, int, int*);
00391   void recvImmediateProxyAll(ProxyDataMsg *);
00392   void recvProxyAll(ProxyDataMsg *);
00393 
00394   static ProxyMgr *Object() { return CkpvAccess(ProxyMgr_instance); }
00395   
00396   int numProxies() { return proxySet.size(); }
00397 
00398   static int nodecount;
00399   ProxyTree &getPtree();
00400  
00401 private:
00402   ProxySet proxySet;
00403   ProxyTree ptree;
00404 
00405   void printProxySpanningTree();
00406 };
00407 
00408 struct ProxyListInfo{
00409         int patchID;
00410         int numProxies;
00411         int *proxyList; //record which PE the proxy is on 
00412 };
00413 
00414 class PatchProxyListMsg: public CMessage_PatchProxyListMsg {
00415 public:
00416         int numPatches;
00417         int *patchIDs;
00418         int *proxyListLen;
00419         int *proxyPEs;
00420 
00421 public:
00422         PatchProxyListMsg(int num) { numPatches = num; }
00423         static PatchProxyListMsg *createPatchProxyListMsg(PatchProxyListMsg **bufs, int bufSize, ProxyListInfo *info, int size);
00424 };
00425 
00426 class NodeProxyMgr : public CBase_NodeProxyMgr
00427 {
00428 private:
00429 /*The following vars are for node-aware spanning tree if NodeProxyMgr is used*/
00430         //Potential TODO: change it to a hashtable if necessary
00431     proxyTreeNode **proxyInfo;
00432     int numPatches;
00433 
00434     CkGroupID localProxyMgr; //a charm Group variable
00435     PatchMap **localPatchMaps;
00436 
00437 /* The following vars are for managing sending proxy list of a patch to PE 0*/
00438         int parentNode; //-1 for root node
00439         int numKidNodes;
00440         int kidRecved;
00441         //about local home patches
00442         int numHomePatches; //the number of homepatches on this node
00443         int homepatchRecved;
00444         ProxyListInfo *localProxyLists;
00445         PatchProxyListMsg **remoteProxyLists;
00446         CmiNodeLock localDepositLock;
00447         CmiNodeLock remoteDepositLock;
00448 
00449 public:
00450     NodeProxyMgr(){
00451         proxyInfo = NULL;
00452         numPatches = 0;
00453         localPatchMaps = new PatchMap *[CkMyNodeSize()];
00454 
00455                 parentNode = -1;
00456                 numKidNodes = 0;
00457                 kidRecved = 0;
00458                 numHomePatches = 0;
00459                 homepatchRecved = 0;
00460                 localProxyLists = NULL;
00461                 remoteProxyLists = NULL;
00462                 localDepositLock = CmiCreateLock();
00463                 remoteDepositLock = CmiCreateLock();
00464     }
00465     ~NodeProxyMgr(){
00466         for(int i=0; i<numPatches; i++) {
00467             delete proxyInfo[i];
00468         }
00469         delete [] proxyInfo;
00470         delete [] localPatchMaps;
00471 
00472                 CmiDestroyLock(localDepositLock);
00473                 CmiDestroyLock(remoteDepositLock);
00474     }
00475 
00476     void createProxyInfo(int numPs){
00477         numPatches = numPs;
00478         proxyInfo = new proxyTreeNode *[numPs];
00479         memset(proxyInfo, 0, sizeof(proxyTreeNode *)*numPs);
00480     }
00481     void registerPatch(int patchID, int numPes, int *pes);
00482     proxyTreeNode *getPatchProxyInfo(int patchID){
00483         return proxyInfo[patchID];
00484     }
00485 
00486     void registerLocalProxyMgr(CkGroupID one){
00487         localProxyMgr = one;
00488     }
00489     const CkGroupID &getLocalProxyMgr(){
00490         return localProxyMgr;
00491     }
00492     void registerLocalPatchMap(int rank, PatchMap *one){
00493         localPatchMaps[rank] = one;
00494     }
00495     PatchMap *getLocalPatchMap(int rank){
00496         return localPatchMaps[rank];
00497     }   
00498 
00499     void recvImmediateProxyData(ProxyDataMsg *msg);
00500     void recvImmediateProxyAll(ProxyDataMsg *msg);
00501     void recvImmediateResults(ProxyCombinedResultRawMsg *);
00502 
00503         //initialize the spanning tree of home patches
00504         void createSTForHomePatches(PatchMap *pmap);
00505         //direct call from local home patches
00506         void sendProxyList(int pid, int *plist, int size);
00507         //remote call to send this node's proxy list info
00508         void sendProxyListInfo(PatchProxyListMsg *msg);
00509         void contributeToParent();
00510 };
00511 
00512 #endif /* PATCHMGR_H */
00513 

Generated on Tue Sep 19 01:17:14 2017 for NAMD by  doxygen 1.4.7