00001
00007 #include "InfoStream.h"
00008 #include "Lattice.h"
00009 #include "ComputeMsmMsa.h"
00010 #include "main.decl.h"
00011 #include "main.h"
00012 #include "ProxyPatch.h"
00013 #include "ProxyMgr.decl.h"
00014 #include "ProxyMgr.h"
00015 #include "AtomMap.h"
00016 #include "PatchMap.h"
00017 #include "Priorities.h"
00018
00019 #define MIN_DEBUG_LEVEL 2
00020
00021 #include "Debug.h"
00022
00023 ProxyPatch::ProxyPatch(PatchID pd) :
00024 Patch(pd), proxyMsgBufferStatus(PROXYMSGNOTBUFFERED),
00025 curProxyMsg(NULL), prevProxyMsg(NULL)
00026 {
00027 DebugM(4, "ProxyPatch(" << pd << ") at " << this << "\n");
00028 ProxyMgr::Object()->registerProxy(patchID);
00029 numAtoms = -1;
00030 parent = -1;
00031
00032 #ifndef NODEAWARE_PROXY_SPANNINGTREE
00033 nChild = 0;
00034 child = new int[proxySpanDim];
00035 #endif
00036
00037 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
00038 localphs = 0;
00039 #ifdef REMOVE_PROXYRESULTMSG_EXTRACOPY
00040 int msgstart = sizeof(envelope)+sizeof(ProxyResultVarsizeMsg);
00041 #else
00042 int msgstart = sizeof(envelope)+sizeof(ProxyResultMsg);
00043 #endif
00044 localphs = CmiCreatePersistent(PatchMap::Object()->node(patchID), 30000, msgstart);
00045 ntreephs = 0;
00046 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00047 treephs = NULL;
00048 #else
00049 treephs = new PersistentHandle[proxySpanDim];
00050 #endif
00051 #endif
00052
00053
00054 #if NAMD_SeparateWaters != 0
00055 numWaterAtoms = -1;
00056 #endif
00057
00058 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00059 depositLock = CmiCreateLock();
00060 #endif
00061 }
00062
00063 ProxyPatch::~ProxyPatch()
00064 {
00065 DebugM(4, "ProxyPatch(" << patchID << ") deleted at " << this << "\n");
00066 ProxyMgr::Object()->unregisterProxy(patchID);
00067
00068
00069
00070 CmiAssert(prevProxyMsg!=NULL);
00071 if(prevProxyMsg!=NULL) {
00072
00073
00074
00075 atomMapper->unregisterIDsCompAtomExt(pExt.begin(),pExt.end());
00076
00077 #if ! CMK_PERSISTENT_COMM || ! USE_PERSISTENT_TREE
00078 delete prevProxyMsg;
00079 #endif
00080 prevProxyMsg = NULL;
00081 }
00082
00083
00084 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00085 #ifdef USE_NODEPATCHMGR
00086 delete [] nodeChildren;
00087 #endif
00088 #endif
00089 delete [] child;
00090
00091 p.resize(0);
00092 pExt.resize(0);
00093
00094 lcpoType.resize(0);
00095
00096 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
00097 CmiDestoryPersistent(localphs);
00098 localphs = 0;
00099 for (int i=0; i<ntreephs; i++) CmiDestoryPersistent(treephs[i]);
00100 delete [] treephs;
00101 #endif
00102 }
00103
00104 void ProxyPatch::boxClosed(int box) {
00105 ProxyGBISP1ResultMsg *msg1;
00106 ProxyGBISP2ResultMsg *msg2;
00107
00108 if (box == 1) {
00109
00110
00111
00112
00113
00114
00115 sendResults();
00116 } else if ( box == 5) {
00117
00118 int msgAtoms = (flags.doNonbonded) ? numAtoms : 0;
00119 msg1 = new (msgAtoms,PRIORITY_SIZE) ProxyGBISP1ResultMsg;
00120 for (int i = 0; i < msgAtoms; i++) {
00121 msg1->psiSum[i] = psiSum[i];
00122 }
00123 msg1->patch = patchID;
00124 msg1->psiSumLen = msgAtoms;
00125 msg1->origPe = CkMyPe();
00126 SET_PRIORITY(msg1,flags.sequence,GB1_PROXY_RESULTS_PRIORITY + PATCH_PRIORITY(patchID));
00127 ProxyMgr::Object()->sendResult(msg1);
00128 } else if ( box == 8) {
00129
00130 int msgAtoms = (flags.doFullElectrostatics) ? numAtoms : 0;
00131 msg2 = new (msgAtoms,PRIORITY_SIZE) ProxyGBISP2ResultMsg;
00132 for (int i = 0; i < msgAtoms; i++) {
00133 msg2->dEdaSum[i] = dEdaSum[i];
00134 }
00135 msg2->patch = patchID;
00136 msg2->dEdaSumLen = msgAtoms;
00137 msg2->origPe = CkMyPe();
00138 SET_PRIORITY(msg2,flags.sequence,GB2_PROXY_RESULTS_PRIORITY + PATCH_PRIORITY(patchID));
00139 ProxyMgr::Object()->sendResult(msg2);
00140 } else if (box == 9) {
00141
00142 } else if (box == 10) {
00143
00144 }
00145
00146
00147 if ( ! --boxesOpen ) {
00148 DebugM(2,patchID << ": " << "Checking message buffer.\n");
00149
00150 if(proxyMsgBufferStatus == PROXYALLMSGBUFFERED) {
00151 CmiAssert(curProxyMsg != NULL);
00152 DebugM(3,"Patch " << patchID << " processing buffered proxy ALL data.\n");
00153 receiveAll(curProxyMsg);
00154 }else if(proxyMsgBufferStatus == PROXYDATAMSGBUFFERED) {
00155 CmiAssert(curProxyMsg != NULL);
00156 DebugM(3,"Patch " << patchID << " processing buffered proxy data.\n");
00157 receiveData(curProxyMsg);
00158 }
00159 } else {
00160 DebugM(3,"ProxyPatch " << patchID << ": " << boxesOpen << " boxes left to close.\n");
00161 }
00162 }
00163
00164
00165 void ProxyPatch::receiveData(ProxyDataMsg *msg)
00166 {
00167 DebugM(3, "receiveData(" << patchID << ")\n");
00168
00169
00170 delete prevProxyMsg;
00171 prevProxyMsg = NULL;
00172
00173 if ( boxesOpen )
00174 {
00175 proxyMsgBufferStatus = PROXYDATAMSGBUFFERED;
00176
00177 curProxyMsg = msg;
00178 return;
00179 }
00180
00181
00182 curProxyMsg = msg;
00183 prevProxyMsg = curProxyMsg;
00184 flags = msg->flags;
00185
00186 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
00187
00188
00189
00190
00191
00192 positionPtrBegin = msg->positionList;
00193 positionPtrEnd = msg->positionList + msg->plLen;
00194 #else
00195 p.resize(msg->plLen);
00196 memcpy(p.begin(), msg->positionList, sizeof(CompAtom)*(msg->plLen));
00197 #endif
00198
00199 #ifdef NAMD_CUDA
00200 cudaAtomPtr = msg->cudaAtomList;
00201 #endif
00202
00203 avgPositionPtrBegin = msg->avgPositionList;
00204 avgPositionPtrEnd = msg->avgPositionList + msg->avgPlLen;
00205
00206
00207 velocityPtrBegin = msg->velocityList;
00208 velocityPtrEnd = msg->velocityList + msg->vlLen;
00209
00210
00211 if ( numAtoms == -1 ) {
00212
00213 numAtoms = msg->plLen;
00214
00215
00216 CmiAssert(msg->plExtLen!=0);
00217 pExt.resize(msg->plExtLen);
00218 memcpy(pExt.begin(), msg->positionExtList, sizeof(CompAtomExt)*(msg->plExtLen));
00219
00220
00221
00222 #if NAMD_SeparateWaters != 0
00223 numWaterAtoms = msg->numWaterAtoms;
00224 #endif
00225
00226 positionsReady(1);
00227 } else {
00228 positionsReady(0);
00229 }
00230 }
00231
00232
00233 void ProxyPatch::receiveAll(ProxyDataMsg *msg)
00234 {
00235 DebugM(3, "receiveAll(" << patchID << ")\n");
00236
00237 if ( boxesOpen )
00238 {
00239 proxyMsgBufferStatus = PROXYALLMSGBUFFERED;
00240 curProxyMsg = msg;
00241 return;
00242 }
00243
00244
00245
00246 if(prevProxyMsg!=NULL) {
00247
00248
00249
00250 atomMapper->unregisterIDsCompAtomExt(pExt.begin(), pExt.end());
00251
00252 }
00253
00254 #if ! CMK_PERSISTENT_COMM || ! USE_PERSISTENT_TREE
00255 delete prevProxyMsg;
00256 #endif
00257 curProxyMsg = msg;
00258 prevProxyMsg = curProxyMsg;
00259
00260 flags = msg->flags;
00261
00262 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
00263
00264
00265
00266
00267
00268 positionPtrBegin = msg->positionList;
00269 positionPtrEnd = msg->positionList + msg->plLen;
00270 #else
00271 p.resize(msg->plLen);
00272 memcpy(p.begin(), msg->positionList, sizeof(CompAtom)*(msg->plLen));
00273 #endif
00274
00275 #ifdef NAMD_CUDA
00276 cudaAtomPtr = msg->cudaAtomList;
00277 #endif
00278
00279 numAtoms = msg->plLen;
00280
00281
00282 avgPositionPtrBegin = msg->avgPositionList;
00283 avgPositionPtrEnd = msg->avgPositionList + msg->avgPlLen;
00284
00285
00286 velocityPtrBegin = msg->velocityList;
00287 velocityPtrEnd = msg->velocityList + msg->vlLen;
00288
00289
00290 if (flags.doGBIS) {
00291 intRad.resize(numAtoms*2);
00292 for (int i = 0; i < numAtoms*2;i++) {
00293 intRad[i] = msg->intRadList[i];
00294 }
00295 }
00296
00297 if (flags.doLCPO) {
00298 lcpoType.resize(numAtoms);
00299 for (int i = 0; i < numAtoms; i++) {
00300 lcpoType[i] = msg->lcpoTypeList[i];
00301 }
00302 }
00303
00304
00305
00306
00307
00308
00309
00310 pExt.resize(msg->plExtLen);
00311 memcpy(pExt.begin(), msg->positionExtList, sizeof(CompAtomExt)*(msg->plExtLen));
00312
00313
00314 #if NAMD_SeparateWaters != 0
00315 numWaterAtoms = msg->numWaterAtoms;
00316 #endif
00317
00318 positionsReady(1);
00319 }
00320
00321 void ProxyPatch::sendResults(void)
00322 {
00323 DebugM(3, "sendResults(" << patchID << ")\n");
00324 register int i = 0;
00325 register ForceList::iterator f_i, f_e, f2_i;
00326 for ( i = Results::normal + 1 ; i <= flags.maxForceMerged; ++i ) {
00327 f_i = f[Results::normal].begin(); f_e = f[Results::normal].end();
00328 f2_i = f[i].begin();
00329 for ( ; f_i != f_e; ++f_i, ++f2_i ) *f_i += *f2_i;
00330 f[i].resize(0);
00331 }
00332 for ( i = flags.maxForceUsed + 1; i < Results::maxNumForces; ++i )
00333 f[i].resize(0);
00334
00335 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
00336 CmiUsePersistentHandle(&localphs, 1);
00337 #endif
00338
00339 if (proxyRecvSpanning == 0) {
00340 #ifdef REMOVE_PROXYRESULTMSG_EXTRACOPY
00341 ProxyResultVarsizeMsg *msg = ProxyResultVarsizeMsg::getANewMsg(CkMyPe(), patchID, PRIORITY_SIZE, f);
00342 #else
00343 ProxyResultMsg *msg = new (PRIORITY_SIZE) ProxyResultMsg;
00344 msg->node = CkMyPe();
00345 msg->patch = patchID;
00346 for ( i = 0; i < Results::maxNumForces; ++i )
00347 msg->forceList[i] = &(f[i]);
00348 #endif
00349 SET_PRIORITY(msg,flags.sequence,PROXY_RESULTS_PRIORITY + PATCH_PRIORITY(patchID));
00350
00351 ProxyMgr::Object()->sendResults(msg);
00352 }
00353 else {
00354 ProxyCombinedResultMsg *msg = new (PRIORITY_SIZE) ProxyCombinedResultMsg;
00355 SET_PRIORITY(msg,flags.sequence,
00356 PROXY_RESULTS_PRIORITY + PATCH_PRIORITY(patchID));
00357 msg->nodes.add(CkMyPe());
00358 msg->patch = patchID;
00359 for ( i = 0; i < Results::maxNumForces; ++i )
00360 msg->forceList[i] = &(f[i]);
00361
00362 ProxyMgr::Object()->sendResults(msg);
00363 }
00364 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
00365 CmiUsePersistentHandle(NULL, 0);
00366 #endif
00367 }
00368
00369 #ifdef NODEAWARE_PROXY_SPANNINGTREE
00370 void ProxyPatch::setSpanningTree(int p, int *c, int n) {
00371 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE && ! defined(USE_NODEPATCHMGR)
00372 if (ntreephs!=0) {
00373 for (int i=0; i<ntreephs; i++) CmiDestoryPersistent(treephs[i]);
00374 delete [] treephs;
00375 }
00376 treephs = NULL;
00377 if (n) {
00378 treephs = new PersistentHandle[n];
00379 for (int i=0; i<n; i++) {
00380 treephs[i] = CmiCreatePersistent(c[i], 27000, sizeof(envelope)+sizeof(ProxyDataMsg));
00381 }
00382 }
00383 ntreephs = n;
00384 #endif
00385 parent=p; nChild = n; nWait = 0;
00386 delete [] child;
00387 if(n==0) {
00388 child = NULL;
00389 return;
00390 }
00391 child = new int[n];
00392 for (int i=0; i<n; i++) child[i] = c[i];
00393
00394 #if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
00395 DebugFileTrace *dft = DebugFileTrace::Object();
00396 dft->openTrace();
00397 dft->writeTrace("ProxyPatch[%d] has %d children: ", patchID, nChild);
00398 for(int i=0; i<nChild; i++)
00399 dft->writeTrace("%d ", child[i]);
00400 dft->writeTrace("\n");
00401 dft->closeTrace();
00402 #endif
00403
00404 }
00405
00406 #ifdef USE_NODEPATCHMGR
00407 void ProxyPatch::setSTNodeChildren(int numNids, int *nids){
00408 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
00409 if (numNodeChild!=0) {
00410 for (int i=0; i<numNodeChild; i++) CmiDestoryPersistent(treephs[i]);
00411 delete [] treephs;
00412 }
00413 treephs = NULL;
00414 if (numNids) {
00415 treephs = new PersistentHandle[numNids];
00416 for (int i=0; i<numNids; i++) {
00417 treephs[i] = CmiCreateNodePersistent(nids[i], 27000, sizeof(envelope)+sizeof(ProxyDataMsg));
00418 }
00419 }
00420 ntreephs = numNids;
00421 #endif
00422 numNodeChild = numNids;
00423 delete [] nodeChildren;
00424 if(numNids==0) {
00425 nodeChildren = NULL;
00426 return;
00427 }
00428 nodeChildren = new int[numNids];
00429 for(int i=0; i<numNids; i++) nodeChildren[i] = nids[i];
00430 }
00431 #endif
00432
00433 #else //branch for NODEAWARE_PROXY_SPANNINGTREE not defined
00434 void ProxyPatch::setSpanningTree(int p, int *c, int n) {
00435 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
00436 if (ntreephs!=0) {
00437 for (int i=0; i<ntreephs; i++) CmiDestoryPersistent(treephs[i]);
00438 }
00439 for (int i=0; i<n; i++) {
00440 treephs[i] = CmiCreatePersistent(c[i], 27000, sizeof(envelope)+sizeof(ProxyDataMsg));
00441 }
00442 ntreephs = n;
00443 #endif
00444 parent=p; nChild = n; nWait = 0;
00445 for (int i=0; i<n; i++) child[i] = c[i];
00446
00447 }
00448 #endif
00449
00450 int ProxyPatch::getSpanningTreeChild(int *c) {
00451 for (int i=0; i<nChild; i++) c[i] = child[i];
00452 return nChild;
00453 }
00454
00455 ProxyCombinedResultMsg *ProxyPatch::depositCombinedResultMsg(ProxyCombinedResultMsg *msg) {
00456 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00457 CmiLock(depositLock);
00458 #endif
00459 nWait++;
00460 if (nWait == 1) msgCBuffer = msg;
00461 else {
00462 NodeIDList::iterator n_i, n_e;
00463 n_i = msg->nodes.begin();
00464 n_e = msg->nodes.end();
00465 for (; n_i!=n_e; ++n_i) msgCBuffer->nodes.add(*n_i);
00466 for ( int k = 0; k < Results::maxNumForces; ++k )
00467 {
00468 register ForceList::iterator r_i;
00469 r_i = msgCBuffer->forceList[k]->begin();
00470 register ForceList::iterator f_i, f_e;
00471 f_i = msg->forceList[k]->begin();
00472 f_e = msg->forceList[k]->end();
00473
00474
00475 int nf = f_e - f_i;
00476 #ifdef ARCH_POWERPC
00477 #pragma disjoint (*f_i, *r_i)
00478 #pragma unroll(4)
00479 #endif
00480 for (int count = 0; count < nf; count++) {
00481 r_i[count].x += f_i[count].x;
00482 r_i[count].y += f_i[count].y;
00483 r_i[count].z += f_i[count].z;
00484 }
00485
00486 }
00487 delete msg;
00488 }
00489
00490 if (nWait == nChild + 1) {
00491 nWait = 0;
00492 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00493 CmiUnlock(depositLock);
00494 #endif
00495
00496 return msgCBuffer;
00497 }
00498
00499 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00500 CmiUnlock(depositLock);
00501 #endif
00502
00503 return NULL;
00504 }
00505
00506
00507 void ProxyPatch::receiveData(ProxyGBISP2DataMsg *msg) {
00508 memcpy(bornRad.begin(), msg->bornRad, sizeof(Real)*numAtoms);
00509 delete msg;
00510 Patch::gbisP2Ready();
00511 }
00512
00513 void ProxyPatch::receiveData(ProxyGBISP3DataMsg *msg) {
00514 memcpy(dHdrPrefix.begin(), msg->dHdrPrefix, sizeof(Real)*msg->dHdrPrefixLen);
00515 delete msg;
00516 Patch::gbisP3Ready();
00517 }
00518
00519 ProxyCombinedResultMsg *ProxyPatch::depositCombinedResultRawMsg(ProxyCombinedResultRawMsg *msg) {
00520 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00521 CmiLock(depositLock);
00522 #endif
00523 nWait++;
00524 if (nWait == 1) msgCBuffer = ProxyCombinedResultMsg::fromRaw(msg);
00525 else {
00526 for (int i=0; i<msg->nodeSize; i++) msgCBuffer->nodes.add(msg->nodes[i]);
00527
00528 register char* isNonZero = msg->isForceNonZero;
00529 register Force* f_i = msg->forceArr;
00530 for ( int k = 0; k < Results::maxNumForces; ++k )
00531 {
00532 register ForceList::iterator r_i;
00533 r_i = msgCBuffer->forceList[k]->begin();
00534 int nf = msg->flLen[k];
00535
00536 #ifdef ARCH_POWERPC
00537 #pragma disjoint (*f_i, *r_i)
00538 #endif
00539 for (int count = 0; count < nf; count++) {
00540 if(*isNonZero){
00541 r_i[count].x += f_i->x;
00542 r_i[count].y += f_i->y;
00543 r_i[count].z += f_i->z;
00544 f_i++;
00545 }
00546 isNonZero++;
00547 }
00548 }
00549 delete msg;
00550 }
00551
00552 if (nWait == nChild + 1) {
00553 nWait = 0;
00554 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00555 CmiUnlock(depositLock);
00556 #endif
00557
00558 return msgCBuffer;
00559 }
00560
00561 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00562 CmiUnlock(depositLock);
00563 #endif
00564
00565 return NULL;
00566 }
00567
00568
00569