Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | File List | Class Members | File Members

ProxyMgr.h

Go to the documentation of this file.
00001 
00007 #ifndef PROXYMGR_H
00008 #define PROXYMGR_H
00009 
00010 
00011 #include "charm++.h"
00012 #include "envelope.h"
00013 
00014 #include "main.h"
00015 #include "NamdTypes.h"
00016 #include "PatchTypes.h"
00017 #include "UniqueSet.h"
00018 #include "UniqueSetIter.h"
00019 #include "ProcessorPrivate.h"
00020 #include "ProxyMgr.decl.h"
00021 
00022 extern int proxySendSpanning, proxyRecvSpanning;
00023 extern const int proxySpanDim;
00024 extern const int inNodeProxySpanDim;
00025 
00026 class RegisterProxyMsg : public CMessage_RegisterProxyMsg {
00027 public:
00028   NodeID node;
00029   PatchID patch;
00030 };
00031 
00032 class UnregisterProxyMsg : public CMessage_UnregisterProxyMsg {
00033 public:
00034   NodeID node;
00035   PatchID patch;
00036 };
00037 
00038 class ProxyAtomsMsg : public CMessage_ProxyAtomsMsg {
00039 public:
00040   PatchID patch;
00041   AtomIDList atomIDList;
00042   static void* pack(ProxyAtomsMsg *msg);
00043   static ProxyAtomsMsg* unpack(void *ptr);
00044 };
00045 
00046 //1. This class represents for both msg types: one that
00047 //is originally known as ProxyAllMsg which is sent
00048 //at the step where atoms migrate; and the other is
00049 //sent during the steps between two migrations.
00050 //2. In the case of memory optimized version, the scenario
00051 //becomes tricky as load balancer will move compute objects
00052 //around so that new ProxyPatches will be created where
00053 //the CompAtomExt list information is not available. If
00054 //the step immediately after the load balancing is a normal
00055 //step, then the CompAtomExt list info has to be resent by
00056 //the HomePatch. Because of the current Proxy msg communication
00057 //scheme where msg is sent to ProxyMgr first, and then retransmitted
00058 //to ProxyPatches, there's overhead when we want to resend CompAtomExt
00059 //list as not all the ProxyPatches that are managed by ProxyMgr are
00060 //newly created ProxyPatches. 
00061 //--Chao Mei
00062 class ProxyDataMsg : public CMessage_ProxyDataMsg {
00063 public:
00064   PatchID patch;
00065   Flags flags;
00066 
00067   int plLen;
00068 
00069   CompAtom *positionList;
00070   int avgPlLen;
00071   CompAtom *avgPositionList;
00072 
00073   //1. The following field will be only
00074   //useful for memory optimized version.
00075   //2. In normal case, adding this field only
00076   //increases the msg length by 4 bytes which
00077   //can be ignored considering the current fast
00078   //communication network
00079   //--Chao Mei
00080   int plExtLen;
00081   CompAtomExt *positionExtList;
00082 
00083 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
00084   //In smp layer, the couter for msg creation and process of communication
00085   //thread is not included in the quiescence detection process. In addition,
00086   //the immediate messages from other nodes are executed on the communication
00087   //thread. If inside the process of immediate messages, some normal Charm++
00088   //messages sent out which will be processed on worker threads. Then QD will
00089   //be a problem that the process of the normal messages sent from communication
00090   //thread is recorded, but the creation of such messages (although recorded
00091   //in the comm thread) is virtually not recorded, i.e., not visible the 
00092   //QD process. So we need to artificially increase the QD counter to 
00093   //compensate for aforementioned msg creation loss.
00094   //The idea is to use the following variable to indicate the normal message
00095   //is sent from the communication thread inside a processing of immediate
00096   //message. If the variable is set, then we should increase the QD counter.
00097   //Chao Mei
00098   char isFromImmMsgCall; //hack for imm msg with QD in SMP 
00099 #endif
00100 
00101   // DMK - Atom Separation (water vs. non-water)
00102   #if NAMD_SeparateWaters != 0
00103     int numWaterAtoms;  // Number of atoms in positionList (from start)
00104                         //   that are part of water hydrogen groups.
00105   #endif
00106 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
00107   //Adding padding bytes to make sure that positionList is
00108   //32-byte aligned which usually gives better cache performance,
00109   //especially on BlueGene/L machine. Otherwise, we have to
00110   //do the extra copy.
00111   //The basic method to calculate padding is to add up
00112   //the size of all the fields so far, including
00113   //the message header (the envelope) , then mod (alignment)
00114   // --Chao Mei
00115 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
00116  #if NAMD_SeparateWaters != 0
00117   char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+sizeof(isFromImmMsgCall)+4*sizeof(int)+3*sizeof(void *))%32)%32];
00118  #else
00119   char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+sizeof(isFromImmMsgCall)+3*sizeof(int)+3*sizeof(void *))%32)%32];
00120  #endif
00121 #else
00122  #if NAMD_SeparateWaters != 0
00123   char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+4*sizeof(int)+3*sizeof(void *))%32)%32];
00124  #else
00125   char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+3*sizeof(int)+3*sizeof(void *))%32)%32];
00126  #endif
00127 #endif
00128 
00129 #endif
00130 
00131 };
00132 
00133 
00134 
00135 class ProxyResultMsg : public CMessage_ProxyResultMsg {
00136 public:
00137   NodeID node;
00138   PatchID patch;
00139   ForceList forceList[Results::maxNumForces];
00140   static void* pack(ProxyResultMsg *msg);
00141   static ProxyResultMsg* unpack(void *ptr);
00142 };
00143 
00144 class ProxyResultVarsizeMsg: public CMessage_ProxyResultVarsizeMsg{
00145 public:
00146     NodeID node;
00147     PatchID patch;
00148     int flLen[Results::maxNumForces];   
00149 
00150     Force *forceArr;
00151     //Indicate the position of the force list that has zero value
00152     //which is not recorded in the above force array.
00153     char *isZero;
00154 
00155     //add padding bytes to make sure the beginning 
00156     //of force arrays is 8-byte aligned as it is originally.
00157     //Therefore, we have to put the forceArr field as
00158     //the first variable of varsize array type
00159     char padding[(8-(sizeof(envelope)+sizeof(NodeID)+sizeof(PatchID)+sizeof(int)*Results::maxNumForces+2*sizeof(void *))%8)%8];   
00160 
00161     //The length of "fls" is Results::maxNumForces
00162     static ProxyResultVarsizeMsg *getANewMsg(NodeID nid, PatchID pid, int prioSize, ForceList *fls); 
00163 };
00164 
00165 class ProxyNodeAwareSpanningTreeMsg: public CMessage_ProxyNodeAwareSpanningTreeMsg{
00166 public:
00167     PatchID patch;
00168     NodeID procID;
00169     int numNodesWithProxies;
00170     int *numPesOfNode;
00171     int *allPes;
00172 
00173     static ProxyNodeAwareSpanningTreeMsg *getANewMsg(PatchID pid, NodeID nid, proxyTreeNode *tree, int size);
00174 
00175     //For debug
00176     void printOut(char *tag);
00177 };
00178 
00179 class ProxyCombinedResultMsg : public CMessage_ProxyCombinedResultMsg {
00180 public:
00181   #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00182   //since this msg may be processed by comm thread in the smp mode,
00183   //this variable helps comm thread to find which proc will actually process it.
00184   NodeID destPe;
00185   #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
00186   //Mainly for QD in the presence of the optimization of using immediate
00187   //message. Refer to the explanation from ProxyDataMsg for the same 
00188   //variable. --Chao Mei
00189   char isFromImmMsgCall;
00190   #endif
00191   #endif
00192   PatchID patch;
00193   NodeIDList nodes;
00194   ForceList forceList[Results::maxNumForces];
00195   static void* pack(ProxyCombinedResultMsg *msg);
00196   static ProxyCombinedResultMsg* unpack(void *ptr);
00197 };
00198 
00199 class ProxySpanningTreeMsg : public CMessage_ProxySpanningTreeMsg {
00200 public:
00201   PatchID patch;
00202   NodeID  node;
00203   NodeIDList tree;
00204   static void* pack(ProxySpanningTreeMsg *msg);
00205   static ProxySpanningTreeMsg* unpack(void *ptr);
00206 };
00207 
00208 class ProxyPatch;
00209 class PatchMap;
00210 
00211 struct ProxyElem {
00212   ProxyElem() : proxyPatch(0) { };
00213   ProxyElem(PatchID pid) : patchID(pid), proxyPatch(0) { };
00214   ProxyElem(PatchID pid, ProxyPatch *p) : patchID(pid), proxyPatch(p) { };
00215 
00216   int hash() const { return patchID; }
00217   int operator==(const ProxyElem & pe) const { return patchID == pe.patchID; }
00218 
00219   PatchID patchID;
00220   ProxyPatch *proxyPatch;
00221 };
00222 
00223 typedef UniqueSet<ProxyElem> ProxySet;
00224 typedef UniqueSetIter<ProxyElem> ProxySetIter;
00225 
00226 class ProxyTree {       // keep track of the spanning trees
00227   public:
00228     int proxyMsgCount;
00229     NodeIDList *proxylist;
00230 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00231     //a node-aware spanning tree array, each element of which
00232     //is a spanning tree for all proxies of a patch
00233     proxyTreeNodeList *naTrees;
00234 #else
00235     NodeIDList *trees;
00236     int *sizes;
00237 #endif
00238     
00239   public:
00240     ProxyTree() {
00241       proxyMsgCount = 0;
00242       proxylist = NULL;
00243 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00244       naTrees = NULL;
00245 #else
00246       trees = NULL;
00247       sizes = NULL;
00248 #endif      
00249     }
00250     ~ProxyTree() {
00251     }
00252 };
00253 
00254 class ProxyMgr : public BOCclass
00255 {
00256 public:
00257   ProxyMgr();
00258   ~ProxyMgr();
00259 
00260   void removeProxies(void);
00261   void removeUnusedProxies(void);
00262   void createProxies(void);
00263 
00264   void createProxy(PatchID pid);
00265   void removeProxy(PatchID pid);
00266 
00267   void registerProxy(PatchID pid);
00268   void recvRegisterProxy(RegisterProxyMsg *);
00269 
00270   void unregisterProxy(PatchID pid);
00271   void recvUnregisterProxy(UnregisterProxyMsg *);
00272 
00273   void setSendSpanning();
00274   int  getSendSpanning();
00275 
00276   void setRecvSpanning();
00277   int  getRecvSpanning();
00278 
00279   void buildProxySpanningTree();
00280   void sendSpanningTrees();
00281   void sendSpanningTreeToHomePatch(int pid, int *tree, int n);
00282   void recvSpanningTreeOnHomePatch(int pid, int *tree, int n);
00283   void sendSpanningTree(ProxySpanningTreeMsg *);
00284   void recvSpanningTree(ProxySpanningTreeMsg *);
00285 
00286   void sendNodeAwareSpanningTreeToHomePatch(int pid, proxyTreeNode *tree, int n);
00287   void recvNodeAwareSpanningTreeOnHomePatch(ProxyNodeAwareSpanningTreeMsg *msg);
00288   void sendNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
00289   void recvNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
00290   //set the proxy patch's parent field
00291   void recvNodeAwareSTParent(int patch, int parent);
00292 
00293   void buildProxySpanningTree2();               // centralized version
00294   void sendProxies(int pid, int *list, int n);
00295   void recvProxies(int pid, int *list, int n);
00296 
00297 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00298   void buildNodeAwareSpanningTree0();
00299   static void buildSinglePatchNodeAwareSpanningTree(PatchID pid, NodeIDList &proxyList, 
00300                                                     proxyTreeNodeList &ptnTree, int *proxyNodeMap);
00301 #else
00302   void buildSpanningTree0();
00303 #endif
00304 
00305   void sendResults(ProxyResultVarsizeMsg *);
00306   void recvResults(ProxyResultVarsizeMsg *);
00307   void sendResults(ProxyResultMsg *);
00308   void recvResults(ProxyResultMsg *);
00309   void sendResults(ProxyCombinedResultMsg *);
00310   void recvResults(ProxyCombinedResultMsg *);
00311   void recvImmediateResults(ProxyCombinedResultMsg *);
00312 
00313   void sendProxyData(ProxyDataMsg *, int, int*);
00314   void recvImmediateProxyData(ProxyDataMsg *);
00315   void recvProxyData(ProxyDataMsg *);
00316 
00317   void sendProxyAll(ProxyDataMsg *, int, int*);
00318   void recvImmediateProxyAll(ProxyDataMsg *);
00319   void recvProxyAll(ProxyDataMsg *);
00320 
00321   static ProxyMgr *Object() { return CkpvAccess(ProxyMgr_instance); }
00322   
00323   int numProxies() { return proxySet.size(); }
00324 
00325   static int nodecount;
00326   ProxyTree &getPtree();
00327  
00328 private:
00329   ProxySet proxySet;
00330   ProxyTree ptree;
00331 
00332   void printProxySpanningTree();
00333 };
00334 
00335 class NodeProxyMgr : public CBase_NodeProxyMgr
00336 {
00337 private:
00338     proxyTreeNode **proxyInfo;
00339     int numPatches;
00340 
00341     CkGroupID localProxyMgr; //a charm Group variable
00342     PatchMap **localPatchMaps;
00343 
00344 public:
00345     NodeProxyMgr(){
00346         proxyInfo = NULL;
00347         numPatches = 0;
00348         localPatchMaps = new PatchMap *[CkMyNodeSize()];
00349     }
00350     ~NodeProxyMgr(){
00351         for(int i=0; i<numPatches; i++) {
00352             delete proxyInfo[i];
00353         }
00354         delete [] proxyInfo;
00355         delete [] localPatchMaps;
00356     }
00357 
00358     void createProxyInfo(int numPs){
00359         numPatches = numPs;
00360         proxyInfo = new proxyTreeNode *[numPs];
00361         memset(proxyInfo, 0, sizeof(proxyTreeNode *)*numPs);
00362     }
00363     void registerPatch(int patchID, int numPes, int *pes);
00364     proxyTreeNode *getPatchProxyInfo(int patchID){
00365         return proxyInfo[patchID];
00366     }
00367 
00368     void registerLocalProxyMgr(CkGroupID one){
00369         localProxyMgr = one;
00370     }
00371     const CkGroupID &getLocalProxyMgr(){
00372         return localProxyMgr;
00373     }
00374     void registerLocalPatchMap(int rank, PatchMap *one){
00375         localPatchMaps[rank] = one;
00376     }
00377     PatchMap *getLocalPatchMap(int rank){
00378         return localPatchMaps[rank];
00379     }   
00380 
00381     void recvImmediateProxyData(ProxyDataMsg *msg);
00382     void recvImmediateProxyAll(ProxyDataMsg *msg);
00383     void recvImmediateResults(ProxyCombinedResultMsg *);
00384 };
00385 
00386 #endif /* PATCHMGR_H */
00387 

Generated on Mon Nov 23 04:59:23 2009 for NAMD by  doxygen 1.3.9.1