NAMD
Public Member Functions | List of all members
NodeProxyMgr Class Reference

#include <ProxyMgr.h>

Inheritance diagram for NodeProxyMgr:

Public Member Functions

 NodeProxyMgr ()
 
 ~NodeProxyMgr ()
 
void createProxyInfo (int numPs)
 
void registerPatch (int patchID, int numPes, int *pes)
 
proxyTreeNodegetPatchProxyInfo (int patchID)
 
void registerLocalProxyMgr (CkGroupID one)
 
const CkGroupID & getLocalProxyMgr ()
 
void registerLocalPatchMap (int rank, PatchMap *one)
 
PatchMapgetLocalPatchMap (int rank)
 
void recvImmediateProxyData (ProxyDataMsg *msg)
 
void recvImmediateProxyAll (ProxyDataMsg *msg)
 
void recvImmediateResults (ProxyCombinedResultRawMsg *)
 
void createSTForHomePatches (PatchMap *pmap)
 
void sendProxyList (int pid, int *plist, int size)
 
void sendProxyListInfo (PatchProxyListMsg *msg)
 
void contributeToParent ()
 

Detailed Description

Definition at line 426 of file ProxyMgr.h.

Constructor & Destructor Documentation

NodeProxyMgr::NodeProxyMgr ( )
inline

Definition at line 450 of file ProxyMgr.h.

References numPatches.

450  {
451  proxyInfo = NULL;
452  numPatches = 0;
453  localPatchMaps = new PatchMap *[CkMyNodeSize()];
454 
455  parentNode = -1;
456  numKidNodes = 0;
457  kidRecved = 0;
458  numHomePatches = 0;
459  homepatchRecved = 0;
460  localProxyLists = NULL;
461  remoteProxyLists = NULL;
462  localDepositLock = CmiCreateLock();
463  remoteDepositLock = CmiCreateLock();
464  }
NodeProxyMgr::~NodeProxyMgr ( )
inline

Definition at line 465 of file ProxyMgr.h.

References numPatches.

465  {
466  for(int i=0; i<numPatches; i++) {
467  delete proxyInfo[i];
468  }
469  delete [] proxyInfo;
470  delete [] localPatchMaps;
471 
472  CmiDestroyLock(localDepositLock);
473  CmiDestroyLock(remoteDepositLock);
474  }

Member Function Documentation

void NodeProxyMgr::contributeToParent ( )

Definition at line 2008 of file ProxyMgr.C.

References PatchProxyListMsg::createPatchProxyListMsg().

Referenced by sendProxyList(), and sendProxyListInfo().

2008  {
2009  if(homepatchRecved!=numHomePatches || kidRecved != numKidNodes) return;
2010 
2011  homepatchRecved = 0;
2012  kidRecved = 0;
2013  //construct the msg
2014  PatchProxyListMsg *msg = PatchProxyListMsg::createPatchProxyListMsg(remoteProxyLists, numKidNodes, localProxyLists, numHomePatches);
2015  if(parentNode == -1) {
2016  //send to proxy mgr on PE[0] as this is the root node
2017  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
2018  cp[0].recvPatchProxyInfo(msg);
2019  }else{
2020  CProxy_NodeProxyMgr cnp(thisgroup);
2021  cnp[parentNode].sendProxyListInfo(msg);
2022  }
2023  for(int i=0; i<numKidNodes; i++) {
2024  delete remoteProxyLists[i];
2025  }
2026 }
static PatchProxyListMsg * createPatchProxyListMsg(PatchProxyListMsg **bufs, int bufSize, ProxyListInfo *info, int size)
Definition: ProxyMgr.C:1882
void NodeProxyMgr::createProxyInfo ( int  numPs)
inline

Definition at line 476 of file ProxyMgr.h.

References numPatches.

476  {
477  numPatches = numPs;
478  proxyInfo = new proxyTreeNode *[numPs];
479  memset(proxyInfo, 0, sizeof(proxyTreeNode *)*numPs);
480  }
void NodeProxyMgr::createSTForHomePatches ( PatchMap pmap)

Definition at line 1922 of file ProxyMgr.C.

References HOMEPATCH_TREE_BRFACTOR, and PatchMap::numPatchesOnNode().

1922  {
1923  //We use implicit tree construction for all home patches
1924  std::vector<int> nodesWithPatches; //record the id of node that has home patches
1925  int myNodeIdx = -1; //the index into the above vector of this node
1926  for(int nodeId=0; nodeId<CkNumNodes(); ++nodeId) {
1927  int hpCnt = 0;
1928  int firstPe = CkNodeFirst(nodeId);
1929  int endPe = firstPe + CkNodeSize(nodeId);
1930  for(int pe=firstPe; pe < endPe; ++pe) {
1931  hpCnt += pmap->numPatchesOnNode(pe);
1932  }
1933  if(hpCnt==0) continue;
1934 
1935  nodesWithPatches.push_back(nodeId);
1936  if(CkMyNode() == nodeId) {
1937  //on my node
1938  myNodeIdx = nodesWithPatches.size()-1;
1939  numHomePatches = hpCnt;
1940  homepatchRecved = 0;
1941  localProxyLists = new ProxyListInfo[hpCnt];
1942  memset(localProxyLists, 0, sizeof(ProxyListInfo)*hpCnt);
1943  }
1944  }
1945 
1946  if(myNodeIdx==-1){
1947  //there's no home patches on this node
1948  //just set to a value that doesn't make sense in spanning tree.
1949  parentNode = -2;
1950  numKidNodes = 0;
1951  kidRecved = 0;
1952  return;
1953  }
1954 
1955  //calculate parent
1956  if(myNodeIdx == 0) {
1957  parentNode = -1;
1958  }else{
1959  int parentIdx = (myNodeIdx-1)/HOMEPATCH_TREE_BRFACTOR;
1960  parentNode = nodesWithPatches[parentIdx];
1961  }
1962 
1963  //calculate kids
1964  numKidNodes = 0;
1965  int totalNodes = nodesWithPatches.size();
1966  for(int i=1; i<=HOMEPATCH_TREE_BRFACTOR; i++) {
1967  int kidId = myNodeIdx*HOMEPATCH_TREE_BRFACTOR+i;
1968  if(kidId >= totalNodes) break;
1969  numKidNodes++;
1970  }
1971  if(numKidNodes!=0) {
1972  remoteProxyLists = new PatchProxyListMsg *[numKidNodes];
1973  }
1974  kidRecved = 0;
1975 
1976  //CkPrintf("Node[%d] has %d homepatches with parent=%d and %d kids \n", CkMyNode(), numHomePatches, parentNode, numKidNodes);
1977 }
#define HOMEPATCH_TREE_BRFACTOR
Definition: ProxyMgr.C:1921
int numPatchesOnNode(int node)
Definition: PatchMap.h:60
PatchMap* NodeProxyMgr::getLocalPatchMap ( int  rank)
inline

Definition at line 495 of file ProxyMgr.h.

495  {
496  return localPatchMaps[rank];
497  }
const CkGroupID& NodeProxyMgr::getLocalProxyMgr ( )
inline

Definition at line 489 of file ProxyMgr.h.

489  {
490  return localProxyMgr;
491  }
proxyTreeNode* NodeProxyMgr::getPatchProxyInfo ( int  patchID)
inline

Definition at line 482 of file ProxyMgr.h.

482  {
483  return proxyInfo[patchID];
484  }
void NodeProxyMgr::recvImmediateProxyAll ( ProxyDataMsg msg)

Definition at line 1746 of file ProxyMgr.C.

References proxyTreeNode::numPes, ProxyDataMsg::patch, PatchMap::patch(), and proxyTreeNode::peIDs.

1746  {
1747 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
1748  CProxy_ProxyMgr cp(localProxyMgr);
1749  proxyTreeNode *ptn = proxyInfo[msg->patch];
1750  CmiAssert(ptn->numPes!=0);
1751  #if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
1752  //This could be executed on comm thd.
1753  printf("NodePMgr::recvImmPAll for patch[%d] on node %d rank %d, prepare to send proc ", msg->patch, CkMyNode(), CkMyRank());
1754  for(int i=0; i<ptn->numPes; i++) {
1755  printf("%d, ", ptn->peIDs[i]);
1756  }
1757  printf("\n");
1758  fflush(stdout);
1759  #endif
1760 
1761  //re-send msg to this nodes's children nodes.
1762  //only the first pe of a node of node-aware ST should contain children nodes
1763  int rank = CkRankOf(ptn->peIDs[0]);
1764  PatchMap *pmap = localPatchMaps[rank];
1765  ProxyPatch *ppatch = (ProxyPatch *)pmap->patch(msg->patch);
1766 
1767  int npid = ppatch->getSTNNodeChild();
1768  int *pids = ppatch->getSTNodeChildPtr();
1769  if(npid>0) {
1770  //only needs to send to other nodes, so check the last entry of pids.
1771  //This is because the data for proxies on the same node have been sent
1772  //later in this function by NodeProxyMgr.
1773  if(pids[npid-1]==CkMyNode()) npid--;
1774  }
1775 
1776 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
1777  if (npid) {
1778  int ntreephs;
1779  PersistentHandle *treephs = ppatch->getSpanningTreePhs(ntreephs);
1780  CmiAssert(treephs && ntreephs >= npid);
1781  CmiUsePersistentHandle(treephs, ntreephs);
1782  }
1783 #endif
1784  CProxy_NodeProxyMgr cnp(thisgroup);
1785  for(int i=0; i<npid; i++) {
1786  ProxyDataMsg *copymsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);
1787  cnp[pids[i]].recvImmediateProxyAll(copymsg);
1788  }
1789 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
1790  CmiUsePersistentHandle(NULL, 0);
1791 #endif
1792 
1793  //re-send msg to it's internal cores
1794 #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
1795  msg->isFromImmMsgCall = (CkMyRank()==CkMyNodeSize());
1796 #endif
1797  cp.recvProxyAll(msg, ptn->numPes, ptn->peIDs);
1798 #else
1799  CkAbort("Bad execution path to NodeProxyMgr::recvImmediateProxyData\n");
1800 #endif
1801 }
Patch * patch(PatchID pid)
Definition: PatchMap.h:235
PatchID patch
Definition: ProxyMgr.h:97
void NodeProxyMgr::recvImmediateProxyData ( ProxyDataMsg msg)

Definition at line 1629 of file ProxyMgr.C.

References proxyTreeNode::numPes, ProxyDataMsg::patch, PatchMap::patch(), and proxyTreeNode::peIDs.

1629  {
1630 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
1631  CProxy_ProxyMgr cp(localProxyMgr);
1632  proxyTreeNode *ptn = proxyInfo[msg->patch];
1633  CmiAssert(ptn->numPes!=0);
1634 
1635  //re-send msg to this nodes's children nodes.
1636  //only the first pe of a node of node-aware ST should contain children nodes
1637  int rank = CkRankOf(ptn->peIDs[0]);
1638  PatchMap *pmap = localPatchMaps[rank];
1639  ProxyPatch *ppatch = (ProxyPatch *)pmap->patch(msg->patch);
1640 
1641  int npid = ppatch->getSTNNodeChild();
1642  int *pids = ppatch->getSTNodeChildPtr();
1643  if(npid>0) {
1644  //only needs to send to other nodes, so check the last entry of pids.
1645  //This is because the data for proxies on the same node have been sent
1646  //later in this function by NodeProxyMgr.
1647  if(pids[npid-1]==CkMyNode()) npid--;
1648  }
1649  CProxy_NodeProxyMgr cnp(thisgroup);
1650 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
1651  if (npid) {
1652  int ntreephs;
1653  PersistentHandle *treephs = ppatch->getSpanningTreePhs(ntreephs);
1654  CmiAssert(treephs && ntreephs >= npid);
1655  CmiUsePersistentHandle(treephs, ntreephs);
1656  }
1657 #endif
1658  for(int i=0; i<npid; i++) {
1659  ProxyDataMsg *copymsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);
1660  cnp[pids[i]].recvImmediateProxyData(copymsg);
1661  }
1662 #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
1663  CmiUsePersistentHandle(NULL, 0);
1664 #endif
1665 
1666  //re-send msg to it's internal cores
1667 #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
1668  msg->isFromImmMsgCall = (CkMyRank()==CkMyNodeSize());
1669 #endif
1670  cp.recvProxyData(msg, ptn->numPes, ptn->peIDs);
1671 #else
1672  CkAbort("Bad execution path to NodeProxyMgr::recvImmediateProxyData\n");
1673 #endif
1674 }
Patch * patch(PatchID pid)
Definition: PatchMap.h:235
PatchID patch
Definition: ProxyMgr.h:97
void NodeProxyMgr::recvImmediateResults ( ProxyCombinedResultRawMsg omsg)

Definition at line 1525 of file ProxyMgr.C.

References ProxyPatch::depositCombinedResultRawMsg(), ProxyPatch::getSpanningTreeParent(), PatchMap::homePatch(), PatchMap::patch(), ProxyCombinedResultRawMsg::patch, and ProxyCombinedResultMsg::toRaw().

1525  {
1526  ProxyCombinedResultRawMsg *msg = omsg;
1527 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
1528  //CkPrintf("recvImmRes called on comm thread%d pe[%d]\n", CkMyRank()==CmiMyNodeSize(), CkMyPe());
1529  //fflush(stdout);
1530 
1531  int destRank = CkRankOf(msg->destPe);
1532  PatchMap *pmap = localPatchMaps[destRank];
1533  HomePatch *home = pmap->homePatch(msg->patch);
1534  if (home) {
1535 #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
1536  msg->isFromImmMsgCall = (CkMyRank()==CkMyNodeSize());
1537 #endif
1538  CProxy_ProxyMgr cp(localProxyMgr);
1539  CmiEnableUrgentSend(1);
1540  cp[msg->destPe].recvResults(msg);
1541  CmiEnableUrgentSend(0);
1542 /*
1543  char *srcfrom = "Isfrom";
1544  if(CkMyRank()!=CmiMyNodeSize()) srcfrom="Notfrom";
1545  CkPrintf("%s comm thread from pe[%d]\n", srcfrom, CkMyPe());
1546  fflush(stdout);
1547 */
1548  }
1549  else {
1550  ProxyPatch *patch = (ProxyPatch *)pmap->patch(msg->patch);
1552  if (ocMsg) {
1553  CProxy_NodeProxyMgr cnp(thisgroup);
1554  ocMsg->destPe = patch->getSpanningTreeParent();
1556  cnp[CkNodeOf(cMsg->destPe)].recvImmediateResults(cMsg);
1557  }
1558  }
1559 #endif
1560 }
Patch * patch(PatchID pid)
Definition: PatchMap.h:235
HomePatch * homePatch(PatchID pid)
Definition: PatchMap.h:240
static ProxyCombinedResultRawMsg * toRaw(ProxyCombinedResultMsg *msg)
Definition: ProxyMgr.C:248
ProxyCombinedResultMsg * depositCombinedResultRawMsg(ProxyCombinedResultRawMsg *)
Definition: ProxyPatch.C:540
int getSpanningTreeParent()
Definition: ProxyPatch.h:33
void NodeProxyMgr::registerLocalPatchMap ( int  rank,
PatchMap one 
)
inline

Definition at line 492 of file ProxyMgr.h.

492  {
493  localPatchMaps[rank] = one;
494  }
void NodeProxyMgr::registerLocalProxyMgr ( CkGroupID  one)
inline

Definition at line 486 of file ProxyMgr.h.

486  {
487  localProxyMgr = one;
488  }
void NodeProxyMgr::registerPatch ( int  patchID,
int  numPes,
int *  pes 
)

Definition at line 1840 of file ProxyMgr.C.

Referenced by ProxyMgr::recvNodeAwareSpanningTree().

1840  {
1841  if(proxyInfo[patchID]) {
1842  delete proxyInfo[patchID];
1843  }
1844  if(numPes == 0) {
1845  proxyInfo[patchID] = NULL;
1846  }else{
1847  proxyInfo[patchID] = new proxyTreeNode(CkNodeOf(pes[0]),numPes,pes);
1848  }
1849 }
void NodeProxyMgr::sendProxyList ( int  pid,
int *  plist,
int  size 
)

Definition at line 1979 of file ProxyMgr.C.

References contributeToParent(), ProxyListInfo::numProxies, ProxyListInfo::patchID, plist, and ProxyListInfo::proxyList.

Referenced by HomePatch::sendProxies().

1979  {
1980  int insertIdx; //indexed from 0
1981  CmiLock(localDepositLock);
1982  insertIdx = homepatchRecved++; //ensure the atomic increment
1983 
1984  localProxyLists[insertIdx].patchID = pid;
1985  localProxyLists[insertIdx].numProxies = size;
1986  localProxyLists[insertIdx].proxyList = plist;
1987 
1988  if(insertIdx == (numHomePatches-1)) {
1989  //all local home patches have contributed
1991  }
1992  CmiUnlock(localDepositLock);
1993 }
static __thread unsigned int * plist
int numProxies
Definition: ProxyMgr.h:410
int * proxyList
Definition: ProxyMgr.h:411
void contributeToParent()
Definition: ProxyMgr.C:2008
void NodeProxyMgr::sendProxyListInfo ( PatchProxyListMsg msg)

Definition at line 1995 of file ProxyMgr.C.

References contributeToParent().

1995  {
1996  int insertIdx; //indexed from 0
1997  CmiLock(localDepositLock);
1998  insertIdx = kidRecved++;
1999 
2000  remoteProxyLists[insertIdx] = msg;
2001  if(insertIdx == (numKidNodes-1)) {
2002  //all kids have contributed;
2004  }
2005  CmiUnlock(localDepositLock);
2006 }
void contributeToParent()
Definition: ProxyMgr.C:2008

The documentation for this class was generated from the following files: