8 #if !defined(WIN32) || defined(__CYGWIN__) 24 #define MIN_DEBUG_LEVEL 5 27 int *PatchMap::nPatchesOnNode = 0;
28 PatchMap::PatchData *PatchMap::patchData = 0;
33 if (CkpvAccess(PatchMap_instance) == 0) {
34 CkpvAccess(PatchMap_instance) =
new PatchMap;
36 return(CkpvAccess(PatchMap_instance));
42 nNodesWithPatches = 0;
43 int npes = CkNumPes();
45 nPatchesOnNode =
new int[npes];
46 memset(nPatchesOnNode,0,npes*
sizeof(
int));
48 computeIdArena = NULL;
56 aDim = bDim = cDim = 0;
57 aAway = bAway = cAway = 1;
58 aPeriodic = bPeriodic = cPeriodic = 0;
59 aMaxIndex = bMaxIndex = cMaxIndex = 0;
64 double maxNumPatches,
int staticAtomAssignment,
65 int asplit,
int bsplit,
int csplit)
67 aPeriodic = lattice.
a_p();
68 bPeriodic = lattice.
b_p();
69 cPeriodic = lattice.
c_p();
75 int minNumPatches = 1;
76 if ( aPeriodic ) minNumPatches *= aAway;
77 if ( bPeriodic ) minNumPatches *= bAway;
78 if ( cPeriodic ) minNumPatches *= cAway;
79 if ( maxNumPatches < minNumPatches ) maxNumPatches = minNumPatches;
83 aDim = (int)(sysDim * aAway / patchSize);
86 aDim = (int)(sysDim * aAway / patchSize);
87 if ((aDim * patchSize) < (sysDim * aAway)) aDim++;
88 if ( aDim < aAway + 1 ) aDim = aAway + 1;
93 bDim = (int)(sysDim * bAway / patchSize);
96 bDim = (int)(sysDim * bAway / patchSize);
97 if ((bDim * patchSize) < (sysDim * bAway)) bDim++;
98 if ( bDim < bAway + 1 ) bDim = bAway + 1;
103 cDim = (int)(sysDim * cAway / patchSize);
106 cDim = (int)(sysDim * cAway / patchSize);
107 if ((cDim * patchSize) < (sysDim * cAway)) cDim++;
108 if ( cDim < cAway + 1 ) cDim = cAway + 1;
111 if ( aDim < 0 || bDim < 0 || cDim < 0 ) {
112 NAMD_die(
"Bug in PatchMap::sizeGrid - negative grid dimension.");
115 if ( staticAtomAssignment ) {
116 if ( aPeriodic || bPeriodic || cPeriodic )
117 NAMD_die(
"Static atom assignment is incompatible with periodic boundary conditions.");
123 const int amin = (aPeriodic ? aAway : 1);
124 const int bmin = (bPeriodic ? bAway : 1);
125 const int cmin = (cPeriodic ? cAway : 1);
129 if ( aDim < amin ) aDim = amin;
130 if ( bDim < bmin ) bDim = bmin;
131 if ( cDim < cmin ) cDim = cmin;
133 if ( maxNumPatches > (
double)aDim*bDim*cDim ) {
134 maxNumPatches = (double)aDim*bDim*cDim;
140 int cdim = maxNumPatches;
141 cdim /= aDim; cdim /= bDim;
142 if ( cdim < cmin ) cdim = cmin;
143 for ( ; cdim <= cDim; ++cdim ) {
144 int bdim = maxNumPatches;
145 bdim /= aDim; bdim /= cdim;
146 if ( bdim < bmin ) bdim = bmin;
147 for ( ; bdim <= bDim; ++bdim ) {
148 int adim = maxNumPatches;
149 adim /= bdim; adim /= cdim;
150 if ( adim < amin ) adim = amin;
151 for ( ; adim <= aDim; ++adim ) {
152 if ( adim*bdim*cdim > maxNumPatches )
break;
154 if ( adim*bdim*cdim > abest*bbest*cbest ) {
155 abest = adim; bbest = bdim; cbest = cdim;
157 if ( abest*bbest*cbest == maxNumPatches )
break;
159 if ( abest*bbest*cbest == maxNumPatches )
break;
161 if ( abest*bbest*cbest == maxNumPatches )
break;
168 return aDim*bDim*cDim;
173 double maxNumPatches,
int staticAtomAssignment,
174 int replicaUniformPatchGrids,
int lcpo,
175 int asplit,
int bsplit,
int csplit)
177 sizeGrid(xmin,xmax,lattice,patchSize,maxNumPatches,staticAtomAssignment,asplit,bsplit,csplit);
179 if ( replicaUniformPatchGrids ) {
180 int oldpcount = aDim * bDim * cDim;
189 int newpcount = aDim * bDim * cDim;
190 if ( newpcount > oldpcount ) {
191 NAMD_bug(
"replicaUniformPatchGrids increased patch count");
193 if ( newpcount < oldpcount ) {
194 iout <<
iINFO <<
"PATCH GRID REDUCED TO BE UNIFORM ACROSS REPLICAS\n";
200 if ( aPeriodic )
iout <<
" (PERIODIC)";
203 if ( bPeriodic )
iout <<
" (PERIODIC)";
206 if ( cPeriodic )
iout <<
" (PERIODIC)";
209 iout << aAway <<
"-AWAY BY ";
210 iout << bAway <<
"-AWAY BY ";
211 iout << cAway <<
"-AWAY\n";
214 aMaxIndex = ( ! aPeriodic || aDim == 2 ) ? 10000 : aDim;
215 bMaxIndex = ( ! bPeriodic || bDim == 2 ) ? 10000 : bDim;
216 cMaxIndex = ( ! cPeriodic || cDim == 2 ) ? 10000 : cDim;
218 aLength = aPeriodic ? 1.0 :
219 ( lcpo || aDim > aAway + 1 ? aDim * (patchSize / aAway) : xmax.
x - xmin.
x );
220 bLength = bPeriodic ? 1.0 :
221 ( lcpo || bDim > bAway + 1 ? bDim * (patchSize / bAway) : xmax.
y - xmin.
y );
222 cLength = cPeriodic ? 1.0 :
223 ( lcpo || cDim > cAway + 1 ? cDim * (patchSize / cAway) : xmax.
z - xmin.
z );
225 aOrigin = aPeriodic ? -0.5 : 0.5 * (xmin.
x + xmax.
x - aLength);
226 bOrigin = bPeriodic ? -0.5 : 0.5 * (xmin.
y + xmax.
y - bLength);
227 cOrigin = cPeriodic ? -0.5 : 0.5 * (xmin.
z + xmax.
z - cLength);
229 nPatches=aDim*bDim*cDim;
230 patchData =
new PatchData[nPatches];
232 patchBounds_a =
new BigReal[2*aDim+1];
233 patchBounds_b =
new BigReal[2*bDim+1];
234 patchBounds_c =
new BigReal[2*cDim+1];
235 for (
int i=0; i<(2*aDim+1); ++i ) {
236 patchBounds_a[i] = ((0.5*(double)i)/(double)aDim) * aLength + aOrigin;
238 for (
int i=0; i<(2*bDim+1); ++i ) {
239 patchBounds_b[i] = ((0.5*(double)i)/(double)bDim) * bLength + bOrigin;
241 for (
int i=0; i<(2*cDim+1); ++i ) {
242 patchBounds_c[i] = ((0.5*(double)i)/(double)cDim) * cLength + cOrigin;
245 for(
int i=0; i<nPatches; ++i)
247 PatchData &p = patchData[i];
253 #ifdef MEM_OPT_VERSION 258 int max_computes = 30;
259 p.cids =
new int[max_computes];
260 for (
int j = 0; j < max_computes; ++j ) p.cids[j] = -1;
261 p.numCidsAllocated = max_computes;
265 myPatch =
new Patch*[nPatches];
267 memset(myPatch,0,nPatches*
sizeof(
Patch*));
268 if ( ! myHomePatch ) {
271 memset(myHomePatch,0,nPatches*
sizeof(
HomePatch*));
277 for (
int i=0; i<nPatches; i++) {
280 if ( myPatch[i]->getPatchID() != i) {
281 DebugM(4,
"patchID("<<myPatch[i]->getPatchID()
287 DebugM(4,
"Patch Count = " <<patchCount<<
"\n");
293 if ( ! CkMyRank() ) {
294 if (patchData && ! computeIdArena ) {
295 for (
int i=0; i<nPatches; i++) {
296 delete [] patchData[i].cids;
300 delete [] nPatchesOnNode;
301 delete computeIdArena;
303 delete [] patchBounds_a;
304 delete [] patchBounds_b;
305 delete [] patchBounds_c;
307 delete [] myHomePatch;
311 #define PACK(type,data) { memcpy(b, &data,sizeof(type)); b += sizeof(type); } 312 #define PACKN(type,data,cnt) { memcpy(b, data,(cnt)*sizeof(type)); b += (cnt)*sizeof(type); } 317 size += 14 *
sizeof(int) + 6 *
sizeof(
BigReal);
318 size += (2*(aDim+bDim+cDim)+3) *
sizeof(
BigReal);
319 size += CkNumPes() *
sizeof(int);
320 for(i=0;i<nPatches;++i)
322 size +=
sizeof(PatchData);
323 size += patchData[i].numCids *
sizeof(
ComputeID);
330 DebugM(4,
"Packing PatchMap on node " << CkMyPe() << std::endl);
336 DebugM(3,
"nPatches = " << nPatches << std::endl);
339 PACK(
int,aPeriodic);
PACK(
int,bPeriodic);
PACK(
int,cPeriodic);
340 PACK(
int,aMaxIndex);
PACK(
int,bMaxIndex);
PACK(
int,cMaxIndex);
343 PACK(
int,nNodesWithPatches);
347 PACKN(
int,nPatchesOnNode,CkNumPes());
348 PACKN(PatchData,patchData,nPatches);
349 for(i=0;i<nPatches;++i) {
350 DebugM(3,
"Packing Patch " << i <<
" is on node " << patchData[i].
node <<
351 " with " << patchData[i].
numCids <<
" cids.\n");
354 if ( buffer + size != b ) {
355 NAMD_bug(
"PatchMap::pack does not match PatchMap::packSize");
361 #define UNPACK(type,data) { memcpy(&data, b, sizeof(type)); b += sizeof(type); } 362 #define UNPACKN(type,data,cnt) { memcpy(data, b, (cnt)*sizeof(type)); b += (cnt)*sizeof(type); } 363 #define SKIPN(type,cnt) { b += (cnt)*sizeof(type); } 367 DebugM(4,
"Unpacking PatchMap on node " << CkMyPe() << std::endl);
370 char *b = (
char*)ptr;
375 nPatches = nPatches_tmp;
377 DebugM(3,
"nPatches = " << nPatches << std::endl);
380 myPatch =
new Patch*[nPatches];
382 memset(myPatch,0,nPatches*
sizeof(
Patch*));
383 if ( ! myHomePatch ) {
386 memset(myHomePatch,0,nPatches*
sizeof(
HomePatch*));
394 UNPACK(
int,nNodesWithPatches);
399 if ( ! patchBounds_a ) patchBounds_a =
new BigReal[2*aDim+1];
400 if ( ! patchBounds_b ) patchBounds_b =
new BigReal[2*bDim+1];
401 if ( ! patchBounds_c ) patchBounds_c =
new BigReal[2*cDim+1];
406 if ( CkMyRank() )
return;
408 UNPACKN(
int,nPatchesOnNode,CkNumPes());
410 if ( ! patchData ) patchData =
new PatchData[nPatches];
411 else if ( ! computeIdArena ) {
412 for(i=0;i<nPatches;++i) {
413 delete [] patchData[i].cids;
416 UNPACKN(PatchData,patchData,nPatches);
418 delete computeIdArena;
422 for(i=0;i<nPatches;++i) {
423 DebugM(3,
"Unpacking Patch " << i <<
" is on node " << patchData[i].
node <<
424 " with " << patchData[i].
numCids <<
" cids.\n");
426 patchData[i].numCidsAllocated = patchData[i].numCids;
434 return CkpvAccess(PatchMap_patchMgr)->homePatches.size();
439 return &(CkpvAccess(PatchMap_patchMgr)->homePatches);
446 for ( i=0; i<nPatches; ++i ) {
447 if ( patchData[i].
node == CkMyPe() ) {
457 for ( i=0; i<nPatches; ++i ) {
458 if ( patchData[i].
basenode == pe ) {
467 if ( nPatchesOnNode[
node] == 0 ) nNodesWithPatches += 1;
468 nPatchesOnNode[
node] += 1;
482 if ( CkNumPes() > 2*nPatches+1 ) {
484 int newnode = ( CkNumPes() +
node - 1 ) % CkNumPes();
487 while ( i < CkNumPes() && !success) {
488 if ( nPatchesOnNode[newnode] == 0 )
492 for (
int count = 0; count <
pid; count ++)
493 if (patchData[count].
basenode > 0 && patchData[count].
basenode == newnode) {
501 newnode = ( CkNumPes() +
node - i - 1 ) % CkNumPes();
504 patchData[
pid].basenode = newnode;
514 if (patchData[
pid].
numCids >= patchData[
pid].numCidsAllocated)
518 patchData[
pid].numCidsAllocated += 10;
519 patchData[
pid].cids =
new int[patchData[
pid].numCidsAllocated];
521 for (i=0; i<patchData[
pid].numCids; i++)
522 patchData[
pid].cids[i] = old[i];
523 for (i=patchData[
pid].
numCids; i<patchData[
pid].numCidsAllocated; i++)
524 patchData[
pid].cids[i] = -1;
527 patchData[
pid].cids[patchData[
pid].numCids]=
cid;
528 patchData[
pid].numCids++;
535 int xinc, yinc, zinc;
538 for(zinc=-1;zinc<=1;zinc++)
540 zi = patchData[
pid].cIndex + zinc;
541 if ((zi < 0) || (zi >= cDim))
542 if ( ! cPeriodic )
continue;
543 for(yinc=-1;yinc<=1;yinc++)
545 yi = patchData[
pid].bIndex + yinc;
546 if ((yi < 0) || (yi >= bDim))
547 if ( ! bPeriodic )
continue;
548 for(xinc=-1;xinc<=1;xinc++)
550 if ((xinc==0) && (yinc==0) && (zinc==0))
553 xi = patchData[
pid].aIndex + xinc;
554 if ((xi < 0) || (xi >= aDim))
555 if ( ! aPeriodic )
continue;
558 neighbor_ids[n]=this->
pid(xi,yi,zi);
562 int xt = 0;
if ( xi < 0 ) xt = -1;
if ( xi >= aDim ) xt = 1;
563 int yt = 0;
if ( yi < 0 ) yt = -1;
if ( yi >= bDim ) yt = 1;
564 int zt = 0;
if ( zi < 0 ) zt = -1;
if ( zi >= cDim ) zt = 1;
572 DebugM(3,
"Patch " <<
pid <<
" has " << n <<
" first neighbors.\n");
582 int xinc, yinc, zinc;
584 const int xs = patchData[
pid].aIndex;
585 const int ys = patchData[
pid].bIndex;
586 const int zs = patchData[
pid].cIndex;
588 for(zinc=0;zinc<=cAway;zinc++)
591 if ((zi < 0) || (zi >= cDim))
592 if ( ! cPeriodic )
continue;
593 for(yinc=(zinc>0 ? -bAway : 0);yinc<=bAway;yinc++)
596 if ((yi < 0) || (yi >= bDim))
597 if ( ! bPeriodic )
continue;
598 for(xinc=((zinc>0 || yinc>0) ? -aAway : 0);xinc<=aAway;xinc++)
600 if ((xinc==0) && (yinc==0) && (zinc==0))
604 if ((xi < 0) || (xi >= aDim))
605 if ( ! aPeriodic )
continue;
607 neighbor_ids[n] = this->
pid(xi,yi,zi);
610 int xt = 0;
if ( xi < 0 ) xt = -1;
if ( xi >= aDim ) xt = 1;
611 int yt = 0;
if ( yi < 0 ) yt = -1;
if ( yi >= bDim ) yt = 1;
612 int zt = 0;
if ( zi < 0 ) zt = -1;
if ( zi >= cDim ) zt = 1;
615 if ( downstream_ids )
617 int xd = ( xi < xs ? xi : xs );
618 int yd = ( yi < ys ? yi : ys );
619 int zd = ( zi < zs ? zi : zs );
620 downstream_ids[n] = this->
pid(xd,yd,zd);
626 DebugM(3,
"Patch " <<
pid <<
" has " << n <<
" second neighbors.\n");
637 int xinc, yinc, zinc;
639 const int xs = patchData[
pid].aIndex;
640 const int ys = patchData[
pid].bIndex;
641 const int zs = patchData[
pid].cIndex;
643 for(zinc=0; zinc<2; zinc++) {
645 for(yinc=0; yinc<2; yinc++) {
647 for(xinc=0; xinc<2; xinc++) {
649 int aIndex =
MODULO(xi,aDim);
650 int bIndex =
MODULO(yi,bDim);
651 int cIndex =
MODULO(zi,cDim);
652 pids[n] = ((cIndex*bDim)+bIndex)*aDim + aIndex;
653 if ( transform_ids ) {
654 int xt = 0;
if ( xi < 0 ) xt = -1;
if ( xi >= aDim ) xt = 1;
655 int yt = 0;
if ( yi < 0 ) yt = -1;
if ( yi >= bDim ) yt = 1;
656 int zt = 0;
if ( zi < 0 ) zt = -1;
if ( zi >= cDim ) zt = 1;
663 DebugM(3,
"Patch " <<
pid <<
" has " << n <<
" second neighbors.\n");
672 int xinc, yinc, zinc;
675 for(zinc=0;zinc<=1;zinc++)
677 zi = patchData[
pid].cIndex + zinc;
678 if ((zi < 0) || (zi >= cDim))
679 if ( ! cPeriodic )
continue;
680 for(yinc=0;yinc<=1;yinc++)
682 yi = patchData[
pid].bIndex + yinc;
683 if ((yi < 0) || (yi >= bDim))
684 if ( ! bPeriodic )
continue;
685 for(xinc=0;xinc<=1;xinc++)
687 if ((xinc==0) && (yinc==0) && (zinc==0))
690 xi = patchData[
pid].aIndex + xinc;
691 if ((xi < 0) || (xi >= aDim))
692 if ( ! aPeriodic )
continue;
695 neighbor_ids[n]=this->
pid(xi,yi,zi);
699 int xt = 0;
if ( xi < 0 ) xt = -1;
if ( xi >= aDim ) xt = 1;
700 int yt = 0;
if ( yi < 0 ) yt = -1;
if ( yi >= bDim ) yt = 1;
701 int zt = 0;
if ( zi < 0 ) zt = -1;
if ( zi >= cDim ) zt = 1;
709 DebugM(3,
"Patch " <<
pid <<
" has " << n <<
" upstream neighbors.\n");
717 int xinc, yinc, zinc;
720 for(zinc=-1;zinc<=0;zinc++)
722 zi = patchData[
pid].cIndex + zinc;
723 if ((zi < 0) || (zi >= cDim))
724 if ( ! cPeriodic )
continue;
725 for(yinc=-1;yinc<=0;yinc++)
727 yi = patchData[
pid].bIndex + yinc;
728 if ((yi < 0) || (yi >= bDim))
729 if ( ! bPeriodic )
continue;
730 for(xinc=-1;xinc<=0;xinc++)
732 if ((xinc==0) && (yinc==0) && (zinc==0))
735 xi = patchData[
pid].aIndex + xinc;
736 if ((xi < 0) || (xi >= aDim))
737 if ( ! aPeriodic )
continue;
740 neighbor_ids[n]=this->
pid(xi,yi,zi);
744 int xt = 0;
if ( xi < 0 ) xt = -1;
if ( xi >= aDim ) xt = 1;
745 int yt = 0;
if ( yi < 0 ) yt = -1;
if ( yi >= bDim ) yt = 1;
746 int zt = 0;
if ( zi < 0 ) zt = -1;
if ( zi >= cDim ) zt = 1;
754 DebugM(3,
"Patch " <<
pid <<
" has " << n <<
" upstream neighbors.\n");
761 CkPrintf(
"---------------------------------------");
762 CkPrintf(
"---------------------------------------\n");
764 CkPrintf(
"nPatches = %d\n",nPatches);
765 for(
int i=0;i<nPatches;i++)
767 CkPrintf(
"Patch %d:\n",i);
768 CkPrintf(
" node = %d\n",patchData[i].
node);
769 CkPrintf(
" xi,yi,zi = %d, %d, %d\n",
770 patchData[i].aIndex,patchData[i].bIndex,patchData[i].cIndex);
771 CkPrintf(
" numCids = %d\n",patchData[i].
numCids);
772 CkPrintf(
" numCidsAllocated = %d\n",patchData[i].numCidsAllocated);
773 for(
int j=0; j < patchData[i].numCids; j++)
775 CkPrintf(
" %10d ",patchData[i].cids[j]);
779 CkPrintf(
"\n---------------------------------------");
780 CkPrintf(
"---------------------------------------\n");
788 if (myHomePatch[
pid] != 0) {
790 <<
"homePatchID("<<
pid<<
") is being re-registered!\n" <<
endi;
792 myHomePatch[
pid] = pptr;
798 if (pptr == myHomePatch[
pid]) {
799 DebugM(4,
"UnregisterHomePatch("<<
pid<<
") at " << pptr <<
"\n");
800 myHomePatch[
pid] = NULL;
807 if (myPatch[
pid] != 0) {
809 <<
"patchID("<<
pid<<
") is being re-registered!\n" <<
endi;
817 if (pptr == myPatch[
pid]) {
818 DebugM(4,
"UnregisterPatch("<<
pid<<
") at " << pptr <<
"\n");
#define UNPACKN(type, data, cnt)
std::ostream & iINFO(std::ostream &s)
Type * getNewArray(int n)
NAMD_HOST_DEVICE Vector c() const
static NAMD_HOST_DEVICE int index(int i=0, int j=0, int k=0)
NAMD_HOST_DEVICE int c_p() const
void basePatchIDList(int pe, PatchIDList &)
std::ostream & iPE(std::ostream &s)
HomePatchList * homePatchList()
std::ostream & endi(std::ostream &s)
int upstreamNeighbors(int pid, PatchID *neighbor_ids)
int index_a(int pid) const
int sizeGrid(ScaledPosition xmin, ScaledPosition xmax, const Lattice &lattice, BigReal patchSize, double maxNumPatches, int staticAtomAssignment, int asplit, int bsplit, int csplit)
int add(const Elem &elem)
NAMD_HOST_DEVICE int b_p() const
void unregisterPatch(PatchID pid, HomePatch *pptr)
void assignBaseNode(PatchID, NodeID)
void newCid(int pid, int cid)
int numCids(int pid) const
void NAMD_bug(const char *err_msg)
int oneAwayNeighbors(int pid, PatchID *neighbor_ids=0)
#define UNPACK(type, data)
void homePatchIDList(PatchIDList &)
int oneOrTwoAwayNeighbors(int pid, PatchID *neighbor_ids, PatchID *downstream_ids=0, int *transform_ids=0)
int index_b(int pid) const
NAMD_HOST_DEVICE int a_p() const
NAMD_HOST_DEVICE Vector a_r() const
NAMD_HOST_DEVICE Vector b_r() const
void NAMD_die(const char *err_msg)
NAMD_HOST_DEVICE Vector c_r() const
NAMD_HOST_DEVICE Vector b() const
void makePatches(ScaledPosition xmin, ScaledPosition xmax, const Lattice &lattice, BigReal patchSize, double maxNumPatches, int staticAtomAssignment, int replicaUniformPatchGrids, int lcpo, int asplit, int bsplit, int csplit)
int basenode(int pid) const
int index_c(int pid) const
int downstreamNeighbors(int pid, PatchID *neighbor_ids)
void replica_min_double(double *dat, int count)
int pid(int aIndex, int bIndex, int cIndex)
void assignNode(PatchID, NodeID)
int cid(int pid, int i) const
void registerPatch(PatchID pid, HomePatch *pptr)
NAMD_HOST_DEVICE Vector a() const
void pack(char *buf, int size)
NAMD_HOST_DEVICE Vector unit(void) const
static PatchMap * Instance()
#define PACKN(type, data, cnt)
int getPatchesInOctet(int pid, PatchID *pids, int *transform_ids=0)