00001
00007 #ifndef PROXYMGR_H
00008 #define PROXYMGR_H
00009
00010
00011 #include "charm++.h"
00012 #include "envelope.h"
00013
00014 #include "main.h"
00015 #include "NamdTypes.h"
00016 #include "PatchTypes.h"
00017 #include "UniqueSet.h"
00018 #include "UniqueSetIter.h"
00019 #include "ProcessorPrivate.h"
00020 #include "ProxyMgr.decl.h"
00021
00022 extern int proxySendSpanning, proxyRecvSpanning;
00023 extern int proxySpanDim;
00024 extern int inNodeProxySpanDim;
00025
00026 #if CMK_PERSISTENT_COMM
00027 #define USE_PERSISTENT_TREE 0
00028 #endif
00029
00030 class ProxyGBISP1ResultMsg: public CMessage_ProxyGBISP1ResultMsg {
00031 public:
00032 int destPe;
00033 int origPe;
00034 PatchID patch;
00035 GBReal *psiSum;
00036 int psiSumLen;
00037 };
00038 class ProxyGBISP2DataMsg: public CMessage_ProxyGBISP2DataMsg {
00039 public:
00040 int origPe;
00041 int destPe;
00042 PatchID patch;
00043 Real *bornRad;
00044 int bornRadLen;
00045 };
00046 class ProxyGBISP2ResultMsg: public CMessage_ProxyGBISP2ResultMsg {
00047 public:
00048 int destPe;
00049 int origPe;
00050 PatchID patch;
00051 GBReal *dEdaSum;
00052 int dEdaSumLen;
00053 };
00054 class ProxyGBISP3DataMsg: public CMessage_ProxyGBISP3DataMsg {
00055 public:
00056 int origPe;
00057 int destPe;
00058 PatchID patch;
00059 Real *dHdrPrefix;
00060 int dHdrPrefixLen;
00061 };
00062
00063 class RegisterProxyMsg : public CMessage_RegisterProxyMsg {
00064 public:
00065 NodeID node;
00066 PatchID patch;
00067 };
00068
00069 class UnregisterProxyMsg : public CMessage_UnregisterProxyMsg {
00070 public:
00071 NodeID node;
00072 PatchID patch;
00073 };
00074
00075
00076
00077
00078
00079
00080
00081
00082
00083
00084
00085
00086
00087
00088
00089
00090
00091 class ProxyDataMsg : public CMessage_ProxyDataMsg {
00092 public:
00093 PatchID patch;
00094 Flags flags;
00095
00096 int plLen;
00097
00098 CompAtom *positionList;
00099 int avgPlLen;
00100 CompAtom *avgPositionList;
00101
00102 int vlLen;
00103 CompAtom *velocityList;
00104
00105
00106 Real *intRadList;
00107
00108 int *lcpoTypeList;
00109
00110
00111
00112
00113
00114
00115
00116
00117 int plExtLen;
00118 CompAtomExt *positionExtList;
00119 CudaAtom *cudaAtomList;
00120
00121 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
00122
00123
00124
00125
00126
00127
00128
00129
00130
00131
00132
00133
00134
00135
00136 char isFromImmMsgCall;
00137 #endif
00138
00139
00140 #if NAMD_SeparateWaters != 0
00141 int numWaterAtoms;
00142
00143 #endif
00144 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
00145
00146
00147
00148
00149
00150
00151
00152
00153 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
00154 #if NAMD_SeparateWaters != 0
00155 char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+sizeof(isFromImmMsgCall)+4*sizeof(int)+3*sizeof(void *))%32)%32];
00156 #else
00157 char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+sizeof(isFromImmMsgCall)+3*sizeof(int)+3*sizeof(void *))%32)%32];
00158 #endif
00159 #else
00160 #if NAMD_SeparateWaters != 0
00161 char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+4*sizeof(int)+3*sizeof(void *))%32)%32];
00162 #else
00163 char padding[(32-(sizeof(envelope)+sizeof(PatchID)+sizeof(Flags)+3*sizeof(int)+3*sizeof(void *))%32)%32];
00164 #endif
00165 #endif
00166
00167 #endif
00168
00169 };
00170
00171
00172
00173 class ProxyResultMsg : public CMessage_ProxyResultMsg {
00174 public:
00175 NodeID node;
00176 PatchID patch;
00177 ForceList *forceList[Results::maxNumForces];
00178 static void* pack(ProxyResultMsg *msg);
00179 static ProxyResultMsg* unpack(void *ptr);
00180 private:
00181 ForceList forceListInternal[Results::maxNumForces];
00182 };
00183
00184 class ProxyResultVarsizeMsg: public CMessage_ProxyResultVarsizeMsg{
00185 public:
00186 NodeID node;
00187 PatchID patch;
00188 int flLen[Results::maxNumForces];
00189
00190 Force *forceArr;
00191
00192
00193 char *isZero;
00194
00195
00196
00197
00198
00199 char padding[(8-(sizeof(envelope)+sizeof(NodeID)+sizeof(PatchID)+sizeof(int)*Results::maxNumForces+2*sizeof(void *))%8)%8];
00200
00201
00202 static ProxyResultVarsizeMsg *getANewMsg(NodeID nid, PatchID pid, int prioSize, ForceList *fls);
00203 };
00204
00205 class ProxyNodeAwareSpanningTreeMsg: public CMessage_ProxyNodeAwareSpanningTreeMsg{
00206 public:
00207 PatchID patch;
00208 NodeID procID;
00209 int numNodesWithProxies;
00210 int *numPesOfNode;
00211 int *allPes;
00212
00213 static ProxyNodeAwareSpanningTreeMsg *getANewMsg(PatchID pid, NodeID nid, proxyTreeNode *tree, int size);
00214
00215
00216 void printOut(char *tag);
00217 };
00218
00219 class ProxyCombinedResultRawMsg : public CMessage_ProxyCombinedResultRawMsg {
00220 public:
00221 int nodeSize;
00222 NodeID *nodes;
00223
00224 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00225
00226
00227 NodeID destPe;
00228 #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
00229
00230
00231
00232 char isFromImmMsgCall;
00233 #endif
00234 #endif
00235 PatchID patch;
00236
00237 int flLen[Results::maxNumForces];
00238 char *isForceNonZero;
00239
00240 Force *forceArr;
00241 };
00242
00243 class ProxyCombinedResultMsg : public CMessage_ProxyCombinedResultMsg {
00244 public:
00245 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00246
00247
00248 NodeID destPe;
00249 #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
00250
00251
00252
00253 char isFromImmMsgCall;
00254 #endif
00255 #endif
00256 PatchID patch;
00257 NodeIDList nodes;
00258 ForceList *forceList[Results::maxNumForces];
00259 static ProxyCombinedResultRawMsg* toRaw(ProxyCombinedResultMsg *msg);
00260 static ProxyCombinedResultMsg* fromRaw(ProxyCombinedResultRawMsg *msg);
00261 private:
00262 ForceList forceListInternal[Results::maxNumForces];
00263 };
00264
00265 class ProxySpanningTreeMsg : public CMessage_ProxySpanningTreeMsg {
00266 public:
00267 PatchID patch;
00268 NodeID node;
00269 NodeIDList tree;
00270 static void* pack(ProxySpanningTreeMsg *msg);
00271 static ProxySpanningTreeMsg* unpack(void *ptr);
00272 };
00273
00274 class ProxyPatch;
00275 class PatchMap;
00276
00277 struct ProxyElem {
00278 ProxyElem() : proxyPatch(0) { };
00279 ProxyElem(PatchID pid) : patchID(pid), proxyPatch(0) { };
00280 ProxyElem(PatchID pid, ProxyPatch *p) : patchID(pid), proxyPatch(p) { };
00281
00282 int hash() const { return patchID; }
00283 int operator==(const ProxyElem & pe) const { return patchID == pe.patchID; }
00284
00285 PatchID patchID;
00286 ProxyPatch *proxyPatch;
00287 };
00288
00289 typedef UniqueSet<ProxyElem> ProxySet;
00290 typedef UniqueSetIter<ProxyElem> ProxySetIter;
00291
00292 class ProxyTree {
00293 public:
00294 int proxyMsgCount;
00295 NodeIDList *proxylist;
00296 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00297
00298
00299 proxyTreeNodeList *naTrees;
00300 #else
00301 NodeIDList *trees;
00302 int *sizes;
00303 #endif
00304
00305 public:
00306 ProxyTree() {
00307 proxyMsgCount = 0;
00308 proxylist = NULL;
00309 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00310 naTrees = NULL;
00311 #else
00312 trees = NULL;
00313 sizes = NULL;
00314 #endif
00315 }
00316 ~ProxyTree() {
00317 }
00318 };
00319
00320 class ProxyMgr : public BOCclass
00321 {
00322 public:
00323 ProxyMgr();
00324 ~ProxyMgr();
00325
00326 void removeProxies(void);
00327 void removeUnusedProxies(void);
00328 void createProxies(void);
00329
00330 void createProxy(PatchID pid);
00331 void removeProxy(PatchID pid);
00332
00333 void registerProxy(PatchID pid);
00334 void recvRegisterProxy(RegisterProxyMsg *);
00335
00336 void unregisterProxy(PatchID pid);
00337 void recvUnregisterProxy(UnregisterProxyMsg *);
00338
00339 void setSendSpanning();
00340 int getSendSpanning();
00341
00342 void setRecvSpanning();
00343 int getRecvSpanning();
00344
00345 void setProxyTreeBranchFactor(int dim);
00346
00347 void buildProxySpanningTree();
00348 void sendSpanningTrees();
00349 void sendSpanningTreeToHomePatch(int pid, int *tree, int n);
00350 void recvSpanningTreeOnHomePatch(int pid, int *tree, int n);
00351 void sendSpanningTree(ProxySpanningTreeMsg *);
00352 void recvSpanningTree(ProxySpanningTreeMsg *);
00353
00354 void sendNodeAwareSpanningTreeToHomePatch(int pid, proxyTreeNode *tree, int n);
00355 void recvNodeAwareSpanningTreeOnHomePatch(ProxyNodeAwareSpanningTreeMsg *msg);
00356 void sendNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
00357 void recvNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
00358
00359 void recvNodeAwareSTParent(int patch, int parent);
00360
00361 void buildProxySpanningTree2();
00362 void sendProxies(int pid, int *list, int n);
00363 void recvProxies(int pid, int *list, int n);
00364 void recvPatchProxyInfo(PatchProxyListMsg *msg);
00365
00366 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00367 void buildNodeAwareSpanningTree0();
00368 static void buildSinglePatchNodeAwareSpanningTree(PatchID pid, NodeIDList &proxyList,
00369 proxyTreeNodeList &ptnTree);
00370 #else
00371 void buildSpanningTree0();
00372 #endif
00373
00374 void sendResults(ProxyResultVarsizeMsg *);
00375 void recvResults(ProxyResultVarsizeMsg *);
00376 void sendResults(ProxyResultMsg *);
00377 void recvResults(ProxyResultMsg *);
00378 void sendResults(ProxyCombinedResultMsg *);
00379
00380 void sendResult(ProxyGBISP1ResultMsg *);
00381 void recvResult(ProxyGBISP1ResultMsg *);
00382 void recvData( ProxyGBISP2DataMsg *);
00383 void sendResult(ProxyGBISP2ResultMsg *);
00384 void recvResult(ProxyGBISP2ResultMsg *);
00385 void recvData( ProxyGBISP3DataMsg *);
00386
00387 void recvResults(ProxyCombinedResultRawMsg *);
00388 void recvImmediateResults(ProxyCombinedResultRawMsg *);
00389
00390 void sendProxyData(ProxyDataMsg *, int, int*);
00391 void recvImmediateProxyData(ProxyDataMsg *);
00392 void recvProxyData(ProxyDataMsg *);
00393
00394 void sendProxyAll(ProxyDataMsg *, int, int*);
00395 void recvImmediateProxyAll(ProxyDataMsg *);
00396 void recvProxyAll(ProxyDataMsg *);
00397
00398 static ProxyMgr *Object() { return CkpvAccess(ProxyMgr_instance); }
00399
00400 int numProxies() { return proxySet.size(); }
00401
00402 static int nodecount;
00403 ProxyTree &getPtree();
00404
00405 private:
00406 ProxySet proxySet;
00407 ProxyTree ptree;
00408
00409 void printProxySpanningTree();
00410 };
00411
00412 struct ProxyListInfo{
00413 int patchID;
00414 int numProxies;
00415 int *proxyList;
00416 };
00417
00418 class PatchProxyListMsg: public CMessage_PatchProxyListMsg {
00419 public:
00420 int numPatches;
00421 int *patchIDs;
00422 int *proxyListLen;
00423 int *proxyPEs;
00424
00425 public:
00426 PatchProxyListMsg(int num) { numPatches = num; }
00427 static PatchProxyListMsg *createPatchProxyListMsg(PatchProxyListMsg **bufs, int bufSize, ProxyListInfo *info, int size);
00428 };
00429
00430 class NodeProxyMgr : public CBase_NodeProxyMgr
00431 {
00432 private:
00433
00434
00435 proxyTreeNode **proxyInfo;
00436 int numPatches;
00437
00438 CkGroupID localProxyMgr;
00439 PatchMap **localPatchMaps;
00440
00441
00442 int parentNode;
00443 int numKidNodes;
00444 int kidRecved;
00445
00446 int numHomePatches;
00447 int homepatchRecved;
00448 ProxyListInfo *localProxyLists;
00449 PatchProxyListMsg **remoteProxyLists;
00450 CmiNodeLock localDepositLock;
00451 CmiNodeLock remoteDepositLock;
00452
00453 public:
00454 NodeProxyMgr(){
00455 proxyInfo = NULL;
00456 numPatches = 0;
00457 localPatchMaps = new PatchMap *[CkMyNodeSize()];
00458
00459 parentNode = -1;
00460 numKidNodes = 0;
00461 kidRecved = 0;
00462 numHomePatches = 0;
00463 homepatchRecved = 0;
00464 localProxyLists = NULL;
00465 remoteProxyLists = NULL;
00466 localDepositLock = CmiCreateLock();
00467 remoteDepositLock = CmiCreateLock();
00468 }
00469 ~NodeProxyMgr(){
00470 for(int i=0; i<numPatches; i++) {
00471 delete proxyInfo[i];
00472 }
00473 delete [] proxyInfo;
00474 delete [] localPatchMaps;
00475
00476 CmiDestroyLock(localDepositLock);
00477 CmiDestroyLock(remoteDepositLock);
00478 }
00479
00480 void createProxyInfo(int numPs){
00481 numPatches = numPs;
00482 proxyInfo = new proxyTreeNode *[numPs];
00483 memset(proxyInfo, 0, sizeof(proxyTreeNode *)*numPs);
00484 }
00485 void registerPatch(int patchID, int numPes, int *pes);
00486 proxyTreeNode *getPatchProxyInfo(int patchID){
00487 return proxyInfo[patchID];
00488 }
00489
00490 void registerLocalProxyMgr(CkGroupID one){
00491 localProxyMgr = one;
00492 }
00493 const CkGroupID &getLocalProxyMgr(){
00494 return localProxyMgr;
00495 }
00496 void registerLocalPatchMap(int rank, PatchMap *one){
00497 localPatchMaps[rank] = one;
00498 }
00499 PatchMap *getLocalPatchMap(int rank){
00500 return localPatchMaps[rank];
00501 }
00502
00503 void recvImmediateProxyData(ProxyDataMsg *msg);
00504 void recvImmediateProxyAll(ProxyDataMsg *msg);
00505 void recvImmediateResults(ProxyCombinedResultRawMsg *);
00506
00507
00508 void createSTForHomePatches(PatchMap *pmap);
00509
00510 void sendProxyList(int pid, int *plist, int size);
00511
00512 void sendProxyListInfo(PatchProxyListMsg *msg);
00513 void contributeToParent();
00514 };
00515
00516 #endif
00517