ProxyMgr Class Reference

#include <ProxyMgr.h>

Inheritance diagram for ProxyMgr:

Public Member Functions
	ProxyMgr ()
	~ProxyMgr ()
void	removeProxies (void)
void	removeUnusedProxies (void)
void	createProxies (void)
void	createProxy (PatchID pid)
void	removeProxy (PatchID pid)
void	registerProxy (PatchID pid)
void	recvRegisterProxy (RegisterProxyMsg *)
void	unregisterProxy (PatchID pid)
void	recvUnregisterProxy (UnregisterProxyMsg *)
void	setSendSpanning ()
int	getSendSpanning ()
void	setRecvSpanning ()
int	getRecvSpanning ()
void	setProxyTreeBranchFactor (int dim)
void	buildProxySpanningTree ()
void	sendSpanningTrees ()
void	sendSpanningTreeToHomePatch (int pid, int *tree, int n)
void	recvSpanningTreeOnHomePatch (int pid, int *tree, int n)
void	sendSpanningTree (ProxySpanningTreeMsg *)
void	recvSpanningTree (ProxySpanningTreeMsg *)
void	sendNodeAwareSpanningTreeToHomePatch (int pid, proxyTreeNode *tree, int n)
void	recvNodeAwareSpanningTreeOnHomePatch (ProxyNodeAwareSpanningTreeMsg *msg)
void	sendNodeAwareSpanningTree (ProxyNodeAwareSpanningTreeMsg *)
void	recvNodeAwareSpanningTree (ProxyNodeAwareSpanningTreeMsg *)
void	recvNodeAwareSTParent (int patch, int parent)
void	buildProxySpanningTree2 ()
void	sendProxies (int pid, int *list, int n)
void	recvProxies (int pid, int *list, int n)
void	recvPatchProxyInfo (PatchProxyListMsg *msg)
void	buildSpanningTree0 ()
void	sendResults (ProxyResultVarsizeMsg *)
void	recvResults (ProxyResultVarsizeMsg *)
void	sendResults (ProxyResultMsg *)
void	recvResults (ProxyResultMsg *)
void	sendResults (ProxyCombinedResultMsg *)
void	sendResult (ProxyGBISP1ResultMsg *)
void	recvResult (ProxyGBISP1ResultMsg *)
void	recvData (ProxyGBISP2DataMsg *)
void	sendResult (ProxyGBISP2ResultMsg *)
void	recvResult (ProxyGBISP2ResultMsg *)
void	recvData (ProxyGBISP3DataMsg *)
void	recvResults (ProxyCombinedResultRawMsg *)
void	recvImmediateResults (ProxyCombinedResultRawMsg *)
void	sendProxyData (ProxyDataMsg *, int, int *)
void	recvImmediateProxyData (ProxyDataMsg *)
void	recvProxyData (ProxyDataMsg *)
void	sendProxyAll (ProxyDataMsg *, int, int *)
void	recvImmediateProxyAll (ProxyDataMsg *)
void	recvProxyAll (ProxyDataMsg *)
int	numProxies ()
ProxyTree &	getPtree ()

Static Public Member Functions
static ProxyMgr *	Object ()

Static Public Attributes
static int	nodecount = 0

Detailed Description

Definition at line 316 of file ProxyMgr.h.

Constructor & Destructor Documentation

ProxyMgr()

ProxyMgr::ProxyMgr

(

)

Definition at line 348 of file ProxyMgr.C.

References NAMD_bug().

                   { 
  if (CkpvAccess(ProxyMgr_instance)) {
    NAMD_bug("Tried to create ProxyMgr twice.");
  }
  CkpvAccess(ProxyMgr_instance) = this;
}

~ProxyMgr()

ProxyMgr::~ProxyMgr

(

)

Definition at line 355 of file ProxyMgr.C.

References removeProxies().

                    { 
  removeProxies();
  CkpvAccess(ProxyMgr_instance) = NULL;
}

Member Function Documentation

buildProxySpanningTree()

void ProxyMgr::buildProxySpanningTree

(

)

Definition at line 558 of file ProxyMgr.C.

References HomePatch::buildSpanningTree(), endi(), PatchMap::homePatch(), PatchMap::homePatchIDList(), iINFO(), iout, PatchMap::Object(), proxyRecvSpanning, proxySendSpanning, proxySpanDim, and ResizeArray< Elem >::size().

Referenced by Node::startup().

{
  PatchIDList pids;
  if (!CkMyPe()) iout << iINFO << "Building spanning tree ... send: " << proxySendSpanning << " recv: " << proxyRecvSpanning 
      << " with branch factor " << proxySpanDim <<"\n" << endi;
  PatchMap::Object()->homePatchIDList(pids);
  for (int i=0; i<pids.size(); i++) {
    HomePatch *home = PatchMap::Object()->homePatch(pids[i]);
    if (home == NULL) CkPrintf("ERROR: homepatch NULL\n");
#ifdef NODEAWARE_PROXY_SPANNINGTREE
    home->buildNodeAwareSpanningTree();
#else
    home->buildSpanningTree();
#endif
  }
}

buildProxySpanningTree2()

void ProxyMgr::buildProxySpanningTree2

(

)

Definition at line 576 of file ProxyMgr.C.

References ResizeArrayIter< T >::begin(), ResizeArrayIter< T >::end(), PatchMap::homePatch(), PatchMap::homePatchIDList(), PatchMap::homePatchList(), PatchMap::Object(), HomePatch::sendProxies(), and ResizeArray< Elem >::size().

Referenced by ComputeMgr::updateLocalComputes5().

{
#if 0
  //The homePatchIDList is an expensive as it goes through
  //every patch ids.
  PatchIDList pids;
  PatchMap::Object()->homePatchIDList(pids);
  for (int i=0; i<pids.size(); i++) {
    HomePatch *home = PatchMap::Object()->homePatch(pids[i]);
    if (home == NULL) CkPrintf("ERROR: homepatch NULL\n");
    home->sendProxies();
  }
#else
  HomePatchList *hpl = PatchMap::Object()->homePatchList();
  HomePatchListIter iter(*hpl);
  for(iter=iter.begin(); iter!=iter.end(); iter++) {
          HomePatch *home = iter->patch;
          home->sendProxies();
  }
#endif
}

buildSpanningTree0()

void ProxyMgr::buildSpanningTree0

(

)

Definition at line 1006 of file ProxyMgr.C.

References ResizeArray< Elem >::find(), MAX_INTERNODE, PatchMap::node(), noInterNode(), PatchMap::numPatches(), numProxies(), PatchMap::Object(), processCpuLoad(), ProxyTree::proxylist, proxyRecvSpanning, proxySendSpanning, proxySpanDim, ResizeArray< Elem >::resize(), ResizeArray< Elem >::setall(), ResizeArray< Elem >::size(), ProxyTree::sizes, ResizeArray< Elem >::swap(), and ProxyTree::trees.

Referenced by Rebalancer::createSpanningTree(), recvPatchProxyInfo(), and recvProxies().

{
        CkPrintf("Info: build spanning tree with send: %d, recv: %d with branch factor %d\n", 
                         proxySendSpanning, proxyRecvSpanning, proxySpanDim);

  int i;

  processCpuLoad();

  int *numPatchesOnNode = new int[CkNumPes()];
  int numNodesWithPatches = 0;
  for (i=0; i<CkNumPes(); i++) numPatchesOnNode[i] = 0;
  int numPatches = PatchMap::Object()->numPatches();
  for (i=0; i<numPatches; i++) {
    int node = PatchMap::Object()->node(i);
    numPatchesOnNode[node]++;
    if (numPatchesOnNode[node] == 1)
      numNodesWithPatches ++;
  }
  int patchNodesLast =
    ( numNodesWithPatches < ( 0.7 * CkNumPes() ) );
  int *ntrees = new int[CkNumPes()];
  for (i=0; i<CkNumPes(); i++) ntrees[i] = 0;
  if (ptree.trees == NULL) ptree.trees = new NodeIDList[numPatches];
  for (int pid=0; pid<numPatches; pid++) 
  {
    int numProxies = ptree.proxylist[pid].size();
    if (numProxies == 0) {
      //CkPrintf ("This is sheer evil!\n\n");
      //ProxyMgr::Object()->sendSpanningTreeToHomePatch(pid, NULL, 0);
      delete [] ntrees;
      delete [] numPatchesOnNode;
      return;
    }
    NodeIDList &tree = ptree.trees[pid];   // spanning tree
    NodeIDList oldtree;  oldtree.swap(tree);
    tree.resize(numProxies+1);
    tree.setall(-1);
    tree[0] = PatchMap::Object()->node(pid);
    int s=1, e=numProxies;
    int nNonPatch = 0;
    int treesize = 1;
    int pp;

    // keep tree persistent for non-intermediate nodes
    for (pp=0; pp<numProxies; pp++) {
      int p = ptree.proxylist[pid][pp];
      int oldindex = oldtree.find(p);
      if (oldindex != -1 && oldindex <= numProxies) {
        int isIntermediate = (oldindex*proxySpanDim+1 <= numProxies);
        if (!isIntermediate) {
          tree[oldindex] = p;
        }
        else if (ntrees[p] < MAX_INTERNODE) {
          tree[oldindex] = p;
          ntrees[p] ++;
        }
      }
    }

    for (pp=0; pp<numProxies; pp++) {
      int p = ptree.proxylist[pid][pp];              // processor number
      if (tree.find(p) != -1) continue;        // already used
      treesize++;
      if (patchNodesLast && numPatchesOnNode[p] ) {
        while (tree[e] != -1) { e--; if (e==-1) e = numProxies; }
        tree[e] = p;
        int isIntermediate = (e*proxySpanDim+1 <= numProxies);
        if (isIntermediate) ntrees[p]++;
      }
      else {
        while (tree[s] != -1) { s++; if (s==numProxies+1) s = 1; }
        int isIntermediate = (s*proxySpanDim+1 <= numProxies);
        if (isIntermediate && (ntrees[p] >= MAX_INTERNODE || noInterNode(p))) {   // TOO MANY INTERMEDIATE TREES
        //if (isIntermediate && ntrees[p] >= MAX_INTERNODE)    // TOO MANY INTERMEDIATE TREES
          while (tree[e] != -1) { e--; if (e==-1) e = numProxies; }
          tree[e] = p;
          isIntermediate = (e*proxySpanDim+1 <= numProxies);
          if (isIntermediate) ntrees[p]++;
        }
        else {
          tree[s] = p;
          nNonPatch++;
          if (isIntermediate) ntrees[p]++;
        }
      }
    }
    // send homepatch's proxy tree
    if(ptree.sizes)
      ptree.sizes[pid] = treesize;
    //ProxyMgr::Object()->sendSpanningTreeToHomePatch(pid, &tree[0], treesize);
  }
  /*for (i=0; i<CkNumPes(); i++) {
    if (ntrees[i] > MAX_INTERNODE) iout << "Processor " << i << "has (guess) " << ntrees[i] << " intermediate nodes." << endi;
  }*/
  delete [] ntrees;
  delete [] numPatchesOnNode;
}

createProxies()

void ProxyMgr::createProxies

(

void

)

Definition at line 416 of file ProxyMgr.C.

References UniqueSet< Elem >::add(), PatchMap::basePatchIDList(), computeNonbondedPairType, computeNonbondedSelfType, PatchMap::MaxOneAway, ComputeMap::node(), PatchMap::node(), ComputeMap::numComputes(), PatchMap::numPatches(), ComputeMap::numPids(), PatchMap::Object(), ComputeMap::Object(), ComputeMap::pid(), PatchMap::registerPatch(), removeProxies(), ResizeArray< Elem >::size(), ComputeMap::type(), and PatchMap::upstreamNeighbors().

Referenced by Node::startup().

{
  // Delete the old proxies.
  removeProxies();

  PatchMap *patchMap = PatchMap::Object();
  int numPatches = patchMap->numPatches();
  int myNode = CkMyPe();
  enum PatchFlag { Unknown, Home, NeedProxy };
  int *patchFlag = new int[numPatches]; 
  int i, j;

  // Note all home patches.
  for ( i = 0; i < numPatches; ++i )
  {
    patchFlag[i] = ( patchMap->node(i) == myNode ) ? Home : Unknown;
  }

#if !(defined(NAMD_CUDA) || defined(NAMD_HIP))
  // Add all upstream neighbors.
  PatchID neighbors[PatchMap::MaxOneAway];
  PatchIDList basepids;
  patchMap->basePatchIDList(myNode,basepids);
  for ( i = 0; i < basepids.size(); ++i )
  {
    if ( patchMap->node(basepids[i]) != myNode ) {
        patchFlag[basepids[i]] = NeedProxy;
    }
    int numNeighbors = patchMap->upstreamNeighbors(basepids[i],neighbors);
    for ( j = 0; j < numNeighbors; ++j )
    {
      if ( ! patchFlag[neighbors[j]] ) {
        patchFlag[neighbors[j]] = NeedProxy;
      }
    }
  }
#endif

  ComputeMap *computeMap = ComputeMap::Object();

  // Check all patch-based compute objects.
  int nc = computeMap->numComputes();
  for ( i = 0; i < nc; ++i )
  {
#if defined(NAMD_CUDA) || defined(NAMD_HIP)
    ComputeType t = computeMap->type(i);
    if ( t == computeNonbondedSelfType || t == computeNonbondedPairType )
      continue;
#elif defined(NAMD_MIC)
    ComputeType t = computeMap->type(i);
    if ( computeMap->directToDevice(i) != 0 ) { continue; } // NOTE: Compute for device will take care of requiring the patch
#endif
    if ( computeMap->node(i) != myNode ) 
      continue;
    int numPid = computeMap->numPids(i);
    for ( j = 0; j < numPid; ++j )
    {
      int pid = computeMap->pid(i,j);
      if ( ! patchFlag[pid] ) {
        patchFlag[pid] = NeedProxy;
      }
    }
  }
  // Create proxy list
  for ( i = 0; i < numPatches; ++i ) {
    if ( patchFlag[i] == NeedProxy )
    { // create proxy patch
      ProxyPatch *proxy = new ProxyPatch(i);
      proxySet.add(ProxyElem(i, proxy));
      patchMap->registerPatch(i, proxy);
    }
  }
  delete[] patchFlag;
}

createProxy()

void ProxyMgr::createProxy

(

PatchID

pid

)

Definition at line 492 of file ProxyMgr.C.

References UniqueSet< Elem >::add(), DebugM, Patch::getPatchID(), PatchMap::Object(), PatchMap::patch(), and PatchMap::registerPatch().

Referenced by ComputeHomeTuples< TholeElem, Thole, TholeValue >::initialize(), and ComputeMgr::updateLocalComputes().

                                 {
  Patch *p = PatchMap::Object()->patch(pid);
  if (!p) {
     DebugM(4,"createProxy("<<pid<<")\n");
     ProxyPatch *proxy = new ProxyPatch(pid);
     proxySet.add(ProxyElem(pid,proxy));
     PatchMap::Object()->registerPatch(pid,proxy);
  }
  else {
     DebugM(4,"createProxy("<<pid<<") found " << p->getPatchID() << "\n");
  }
    
}

getPtree()

ProxyTree & ProxyMgr::getPtree

(

)

Definition at line 384 of file ProxyMgr.C.

Referenced by Rebalancer::createSpanningTree(), Rebalancer::decrSTLoad(), and Rebalancer::incrSTLoad().

                              {
  return ptree;
}

getRecvSpanning()

int ProxyMgr::getRecvSpanning

(

)

Definition at line 375 of file ProxyMgr.C.

References proxyRecvSpanning.

Referenced by Node::startup().

                              {
  return proxyRecvSpanning;
}

getSendSpanning()

int ProxyMgr::getSendSpanning

(

)

Definition at line 366 of file ProxyMgr.C.

References proxySendSpanning.

Referenced by Node::startup().

                              {
  return proxySendSpanning;
}

numProxies()

int ProxyMgr::numProxies ( )

inline

Definition at line 396 of file ProxyMgr.h.

Referenced by buildSpanningTree0(), and sendSpanningTrees().

{ return proxySet.size(); }

Object()

static ProxyMgr* ProxyMgr::Object ( )

inlinestatic

Definition at line 394 of file ProxyMgr.h.

Referenced by ProxyPatch::boxClosed(), Rebalancer::createSpanningTree(), Rebalancer::decrSTLoad(), Rebalancer::incrSTLoad(), ComputeHomeTuples< TholeElem, Thole, TholeValue >::initialize(), Sync::openSync(), HomePatch::positionsReady(), HomePatch::positionsReady_SOA(), ProxyPatch::ProxyPatch(), recvImmediateProxyAll(), recvImmediateProxyData(), recvNodeAwareSpanningTree(), recvSpanningTree(), WorkDistrib::savePatchMap(), WorkDistrib::sendPatchMap(), HomePatch::sendProxies(), HomePatch::sendSpanningTree(), sendSpanningTrees(), Node::startup(), ComputeMgr::updateLocalComputes5(), and ProxyPatch::~ProxyPatch().

{ return CkpvAccess(ProxyMgr_instance); }

recvData() [1/2]

void ProxyMgr::recvData

(

ProxyGBISP2DataMsg *

msg

)

Definition at line 1861 of file ProxyMgr.C.

References PatchMap::Object(), ProxyGBISP2DataMsg::patch, PatchMap::patch(), and ProxyPatch::receiveData().

                                                 {  //hp -d> pp
  ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
  proxy->receiveData(msg); // deleted in ProxyPatch::receiveAtoms() ?
}

recvData() [2/2]

void ProxyMgr::recvData

(

ProxyGBISP3DataMsg *

msg

)

Definition at line 1876 of file ProxyMgr.C.

References PatchMap::Object(), ProxyGBISP3DataMsg::patch, PatchMap::patch(), and ProxyPatch::receiveData().

                                                 {   //hp -d> pp
  ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
  proxy->receiveData(msg); // deleted in ProxyPatch::receiveAtoms() ?
}

recvImmediateProxyAll()

void ProxyMgr::recvImmediateProxyAll

(

ProxyDataMsg *

msg

)

Definition at line 1710 of file ProxyMgr.C.

References ProxyPatch::getSpanningTreeChildPtr(), ProxyPatch::getSpanningTreeNChild(), PatchMap::Object(), Object(), ProxyDataMsg::patch, PatchMap::patch(), proxySendSpanning, and sendProxyAll().

                                                 {
  ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
  #if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
  DebugFileTrace *dft = DebugFileTrace::Object();
  dft->openTrace();
  dft->writeTrace("PMgr::recvImmPAll for patch[%d]\n", msg->patch);
  CmiAssert(proxy!=NULL);
  dft->writeTrace("PMgr::recvImmPAll assertion OK for patch[%d]\n", msg->patch);
  dft->closeTrace();
  #endif
  if (proxySendSpanning == 1) {
    // copy the message and send to spanning children
    //int *pids = (int*)alloca(proxy->getSpanningTreeNChild()*sizeof(int));
    //int npid = proxy->getSpanningTreeChild(pids);
    int npid = proxy->getSpanningTreeNChild();
    int *pids = (int *)proxy->getSpanningTreeChildPtr();
    if (npid) {
        ProxyDataMsg *newmsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);      
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
        int ntreephs;
        PersistentHandle *treephs = proxy->getSpanningTreePhs(ntreephs);
        CmiAssert(treephs && ntreephs == npid);
        CmiUsePersistentHandle(treephs, ntreephs);
#endif
        ProxyMgr::Object()->sendProxyAll(newmsg,npid,pids);
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
        CmiUsePersistentHandle(NULL, 0);
#endif
    }
  }
  /* send to self via EP method to preserve priority */
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp[CkMyPe()].recvProxyAll(msg);
}

recvImmediateProxyData()

void ProxyMgr::recvImmediateProxyData

(

ProxyDataMsg *

msg

)

Definition at line 1595 of file ProxyMgr.C.

References ProxyPatch::getSpanningTreeChildPtr(), ProxyPatch::getSpanningTreeNChild(), PatchMap::Object(), Object(), ProxyDataMsg::patch, PatchMap::patch(), proxySendSpanning, and sendProxyData().

                                                  {
  ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);  
  if (proxySendSpanning == 1) {
    // copy the message and send to spanning children
    //int *pids = (int*)alloca(proxy->getSpanningTreeNChild()*sizeof(int));
    //int npid = proxy->getSpanningTreeChild(pids);
    int npid = proxy->getSpanningTreeNChild();
    int *pids = (int *)proxy->getSpanningTreeChildPtr();
    if (npid) {        
        ProxyDataMsg *newmsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);     
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
        int ntreephs;
        PersistentHandle *treephs = proxy->getSpanningTreePhs(ntreephs);
        CmiAssert(treephs && ntreephs == npid);
        CmiUsePersistentHandle(treephs, ntreephs);
#endif
        ProxyMgr::Object()->sendProxyData(newmsg,npid,pids);
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
        CmiUsePersistentHandle(NULL, 0);
#endif
      #if 0
      //ChaoMei: buggy code??? the spanning tree doesn't always have 2 levels
      //At the second level of the tree immediate messages are not needed
      CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
      cp.recvProxyData(newmsg,npid,pids);
      #endif
    }
  }
  /* send to self via EP method to preserve priority */
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp[CkMyPe()].recvProxyData(msg);
}

recvImmediateResults()

void ProxyMgr::recvImmediateResults

(

ProxyCombinedResultRawMsg *

omsg

)

Definition at line 1505 of file ProxyMgr.C.

References ProxyPatch::depositCombinedResultRawMsg(), ProxyPatch::getSpanningTreeParent(), PatchMap::homePatch(), PatchMap::Object(), PatchMap::patch(), ProxyCombinedResultRawMsg::patch, and ProxyCombinedResultMsg::toRaw().

                                                                   {
  HomePatch *home = PatchMap::Object()->homePatch(omsg->patch);
  if (home) {
    CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);        
    CmiEnableUrgentSend(1);
    cp[CkMyPe()].recvResults(omsg);
    CmiEnableUrgentSend(0);
  }
  else {
    ProxyPatch *patch = (ProxyPatch *)PatchMap::Object()->patch(omsg->patch);
        ProxyCombinedResultMsg *ocMsg = patch->depositCombinedResultRawMsg(omsg);
    if (ocMsg) {
                CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
                ProxyCombinedResultRawMsg *cMsg = ProxyCombinedResultMsg::toRaw(ocMsg);         
                cp[patch->getSpanningTreeParent()].recvImmediateResults(cMsg);
    }
  }
}

recvNodeAwareSpanningTree()

void ProxyMgr::recvNodeAwareSpanningTree

(

ProxyNodeAwareSpanningTreeMsg *

msg

)

Definition at line 1229 of file ProxyMgr.C.

References ResizeArray< Elem >::add(), ALLOCA, ProxyNodeAwareSpanningTreeMsg::allPes, PatchMap::homePatch(), ResizeArray< Elem >::item(), nodecount, ProxyNodeAwareSpanningTreeMsg::numNodesWithProxies, ProxyNodeAwareSpanningTreeMsg::numPesOfNode, PatchMap::Object(), Object(), PatchMap::patch(), ProxyNodeAwareSpanningTreeMsg::patch, ProxyNodeAwareSpanningTreeMsg::printOut(), ProxyNodeAwareSpanningTreeMsg::procID, proxySpanDim, NodeProxyMgr::registerPatch(), sendNodeAwareSpanningTree(), ProxyPatch::setSpanningTree(), and ResizeArray< Elem >::size().

                                                                          {
#if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
    DebugFileTrace *dft = DebugFileTrace::Object();
    dft->openTrace();
    dft->writeTrace("PMgr::recvST0 for patch[%d] with #nodes=%d\n", msg->patch, msg->numNodesWithProxies);
    dft->closeTrace();
    msg->printOut("PMgr::recvST");
#endif

    //This function is divided into three parts. The tree root is msg->allPes[0]
    //1. set up its own immediate childrens
    int treesize = msg->numNodesWithProxies; //at least include one as its internal procs    
    int iNChild = msg->numPesOfNode[0]-1; //number of internal children
    int eNChild = treesize-1; //number of external children

    CmiAssert(treesize>0);
    //use the same way of computing children in HomePatch::setupChildrenFromProxySpanningTree    
    eNChild = (proxySpanDim>eNChild)?eNChild:proxySpanDim;
    int iSlots = proxySpanDim-eNChild;
    if(iNChild>0) {
        if(iSlots==0){
            //at least having one internal child
            iNChild = 1;    
        }else{
            iNChild = (iSlots>iNChild)?iNChild:iSlots;
        }
    }    
    int numChild = iNChild + eNChild;
    if(numChild==0){
        //Indicating this proxy is a leaf in the spanning tree
        ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
        proxy->setSpanningTree(msg->procID, NULL, 0);
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
                //When using NODEPATCHMGR, the proc-level is a flat list attached to the node
                //while the node-level spanning tree obeys the branch factor.
                //As a result, when passing down spanning trees, if this proc is on the same node
                //of its parent, then the NodeProxyMgr has already been set by its parent. There's
                //no need resetting here. However, the nodeChildren attached to this proxy has
                //to be set to NULL. -Chao Mei
                int onSameNode = (CkMyNode() == CkNodeOf(msg->procID));
        //set up proxyInfo inside NodeProxyMgr
        if(!onSameNode && !PatchMap::Object()->homePatch(msg->patch)){
            //only when this processor contains a proxy patch of "msg->patch"
            //is the patch registeration in NodeProxyMgr needed,
            //and itself needs to be registered
            CProxy_NodeProxyMgr pm(CkpvAccess(BOCclass_group).nodeProxyMgr);
            NodeProxyMgr *npm = pm[CkMyNode()].ckLocalBranch();        
            npm->registerPatch(msg->patch, msg->numPesOfNode[0], msg->allPes);            
        }
        //set children in terms of node ids
        proxy->setSTNodeChildren(0, NULL);       
#endif
        delete msg;
        return;
    }

    nodecount++;
    //if (nodecount > MAX_INTERNODE) 
    //  iout << "Processor " << CkMyPe() << "has (actual) " << nodecount << " intermediate nodes." << endi;

    if(!PatchMap::Object()->homePatch(msg->patch)){
        //the home patch of this spanning tree has been already set
        //in HomePatch::setupChildrenFromProxySpanningTree, so we
        //only care about the children setup for proxy patches here
        ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
        ALLOCA(int,children,numChild);
        //add external children
        int *p = msg->allPes + msg->numPesOfNode[0];
        for(int i=0; i<eNChild; i++) {
            children[i] = *p;
            p += msg->numPesOfNode[i+1];
        }
        //add internal children
        for(int i=eNChild, j=1; i<numChild; i++, j++) {
            children[i] = msg->allPes[j]; 
        }
        proxy->setSpanningTree(msg->procID, children, numChild);

#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
                int onSameNode = (CkMyNode() == CkNodeOf(msg->procID));
                if(!onSameNode) {
                        //set up proxyInfo inside NodeProxyMgr
                        CProxy_NodeProxyMgr pm(CkpvAccess(BOCclass_group).nodeProxyMgr);
                        NodeProxyMgr *npm = pm[CkMyNode()].ckLocalBranch();        
                        npm->registerPatch(msg->patch, msg->numPesOfNode[0], msg->allPes);        
        
                        //set children in terms of node ids
                        ALLOCA(int,nodeChildren,eNChild+1);
                        p = msg->allPes + msg->numPesOfNode[0];
                        for(int i=0; i<eNChild; i++) {
                                nodeChildren[i] = CkNodeOf(*p);
                                p += msg->numPesOfNode[i+1];
                        }
                        //the last entry always stores the node id that contains this proxy
                        nodeChildren[eNChild] = CkNodeOf(msg->allPes[0]);
                        proxy->setSTNodeChildren(eNChild+1, nodeChildren);
                } else {
                        proxy->setSTNodeChildren(0, NULL);
                }
#endif
    }

    //2. send msgs for the tree to children proxies
    ResizeArray<int> *exTreeChildSize = new ResizeArray<int>[numChild];
    ResizeArray<int *> *exTreeChildPtr = new ResizeArray<int *>[numChild];    

    //2a. first processing children of external nodes
    if(eNChild > 0) {    
        int nodesToCnt = 1; //the number of children each root (current root's 
                            //immedidate external nodes) has in each level
        int pos = 1; //track the iteration over msg->numPesOfNode and skip the current root
        int *pePtr = msg->allPes + msg->numPesOfNode[0];
        int done = 0;
        while(!done) {
            for(int childID=0; childID<eNChild; childID++) {
                //iterate nodes on each level
                for(int i=0; i<nodesToCnt; i++) {
                    int cursize = msg->numPesOfNode[pos];
                    exTreeChildSize[childID].add(cursize);
                    exTreeChildPtr[childID].add(pePtr);
                    pos++;
                    pePtr += cursize; 
                    if(pos==msg->numNodesWithProxies) {
                        done = 1;
                        break;
                    }
                }
                if(done) break;                         
            }
            nodesToCnt *= proxySpanDim;
        }
    }

    //2b. secondly processing children on the same node
    if(iNChild>0) {
        int nodesToCnt = 1; //the number of children each root (current root's 
                            //immedidate internal child proxies) has in each level
        int pos = 1; //track the iteration over proxies on the same node and skip the current root
        int *pePtr = msg->allPes+1; //skip the root
        int done = 0;
        while(!done) {
            for(int childID=eNChild; childID<numChild; childID++) {
                //iterate nodes on each level
                for(int i=0; i<nodesToCnt; i++) {                    
                    exTreeChildSize[childID].add(1);
                    exTreeChildPtr[childID].add(pePtr);
                    pos++;
                    pePtr++; 
                    if(pos==msg->numPesOfNode[0]) {
                        done = 1;
                        break;
                    }
                }
                if(done) break;                         
            }
            nodesToCnt *= proxySpanDim;
        }
    }
          
    for(int i=0; i<numChild; i++) {                
        ResizeArray<int> *allSizes = &exTreeChildSize[i];
        ResizeArray<int *> *allPtrs = &exTreeChildPtr[i];
        int totalNodes = allSizes->size();
        int totalPes = 0;
        for(int j=0; j<totalNodes; j++) totalPes += allSizes->item(j);
        ProxyNodeAwareSpanningTreeMsg *cmsg = new(totalNodes, totalPes, 0) ProxyNodeAwareSpanningTreeMsg;
        cmsg->patch = msg->patch;
        cmsg->procID = CkMyPe();
        cmsg->numNodesWithProxies = totalNodes;
        int *pAllPes = cmsg->allPes;
        for(int j=0; j<totalNodes; j++) {
            int numPes = allSizes->item(j);
            cmsg->numPesOfNode[j] = numPes;
            memcpy(pAllPes, allPtrs->item(j), sizeof(int)*numPes);
            pAllPes += numPes;
        }
        #if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
        cmsg->printOut("sndChi:");
        #endif
        ProxyMgr::Object()->sendNodeAwareSpanningTree(cmsg);
    }
    delete [] exTreeChildSize;
    delete [] exTreeChildPtr;  
    delete msg;
}

recvNodeAwareSpanningTreeOnHomePatch()

void ProxyMgr::recvNodeAwareSpanningTreeOnHomePatch

(

ProxyNodeAwareSpanningTreeMsg *

msg

)

Definition at line 1146 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), ProxyNodeAwareSpanningTreeMsg::patch, and HomePatch::recvNodeAwareSpanningTree().

{
  HomePatch *p = PatchMap::Object()->homePatch(msg->patch);
  p->recvNodeAwareSpanningTree(msg);
  delete msg;
}

recvNodeAwareSTParent()

void ProxyMgr::recvNodeAwareSTParent	(	int	patch,
		int	parent
	)

Definition at line 1415 of file ProxyMgr.C.

References PatchMap::Object(), PatchMap::patch(), and ProxyPatch::setSpanningTree().

                                                         {
#if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
    DebugFileTrace *dft = DebugFileTrace::Object();
    dft->openTrace();
    dft->writeTrace("PMgr::recvSTParent: for ProxyPatch[%d], parent is %d\n", patch, parent);
    dft->closeTrace();
#endif
    ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(patch);
    CmiAssert(proxy!=NULL);
    proxy->setSpanningTree(parent, NULL, 0);
}

recvPatchProxyInfo()

void ProxyMgr::recvPatchProxyInfo

(

PatchProxyListMsg *

msg

)

Definition at line 658 of file ProxyMgr.C.

References buildSpanningTree0(), PatchMap::numPatches(), PatchProxyListMsg::numPatches, PatchMap::Object(), PatchProxyListMsg::patchIDs, ProxyTree::proxylist, PatchProxyListMsg::proxyListLen, ProxyTree::proxyMsgCount, PatchProxyListMsg::proxyPEs, ResizeArray< Elem >::resize(), and sendSpanningTrees().

                                                       {
        int nPatches = PatchMap::Object()->numPatches();
        if(ptree.proxylist == NULL) ptree.proxylist = new NodeIDList[nPatches];
        CmiAssert(msg->numPatches == nPatches);
        int peIdx = 0;
        for(int i=0; i<nPatches; i++) {
                int pid = msg->patchIDs[i];
                int plen = msg->proxyListLen[i];
                ptree.proxylist[pid].resize(plen);
                for(int j=0; j<plen; j++) {
                        ptree.proxylist[pid][j] = msg->proxyPEs[peIdx++];
                }               
        }
        delete msg;
        
        //debug
        //outputProxyTree(ptree, nPatches);

        ptree.proxyMsgCount = 0;
    // building and sending of trees is done in two steps now
    // so that the building step can be shifted to the load balancer
#ifdef NODEAWARE_PROXY_SPANNINGTREE
    buildNodeAwareSpanningTree0();
#else
    buildSpanningTree0();
#endif
    sendSpanningTrees();
}

recvProxies()

void ProxyMgr::recvProxies	(	int	pid,
		int *	list,
		int	n
	)

Definition at line 633 of file ProxyMgr.C.

References buildSpanningTree0(), PatchMap::numPatches(), PatchMap::Object(), ProxyTree::proxylist, ProxyTree::proxyMsgCount, ResizeArray< Elem >::resize(), and sendSpanningTrees().

{
  int nPatches = PatchMap::Object()->numPatches();
  if (ptree.proxylist == NULL)
    ptree.proxylist = new NodeIDList[nPatches];
  ptree.proxylist[pid].resize(n);
  for (int i=0; i<n; i++)
    ptree.proxylist[pid][i] = list[i];
  ptree.proxyMsgCount ++;
  if (ptree.proxyMsgCount == nPatches) {
        //debug
        //outputProxyTree(ptree, nPatches);

    ptree.proxyMsgCount = 0;
    // building and sending of trees is done in two steps now
    // so that the building step can be shifted to the load balancer
#ifdef NODEAWARE_PROXY_SPANNINGTREE
    buildNodeAwareSpanningTree0();
#else
    buildSpanningTree0();    
#endif
    sendSpanningTrees();
  }
}

recvProxyAll()

void ProxyMgr::recvProxyAll

(

ProxyDataMsg *

msg

)

Definition at line 1693 of file ProxyMgr.C.

References PatchMap::Object(), ProxyDataMsg::patch, PatchMap::patch(), proxySendSpanning, and ProxyPatch::receiveAll().

                                        {
//Chao Mei: hack for QD in case of SMP with immediate msg
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
    if(proxySendSpanning && msg->isFromImmMsgCall){
//    CkPrintf("qdcreate called on pe[%d]\n", CkMyPe());
//    fflush(stdout);
        //To compensate for the counter loss for message creation
        //inside the process of immediate message on comm thread
        CkpvAccess(_qd)->create();
    }
#endif

  ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
  proxy->receiveAll(msg); // deleted in ProxyPatch::receiveAtoms()
}

recvProxyData()

void ProxyMgr::recvProxyData

(

ProxyDataMsg *

msg

)

Definition at line 1579 of file ProxyMgr.C.

References PatchMap::Object(), ProxyDataMsg::patch, PatchMap::patch(), proxySendSpanning, and ProxyPatch::receiveData().

                                         {
//Chao Mei: hack for QD in case of SMP with immediate msg
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
    if(proxySendSpanning && msg->isFromImmMsgCall){
//    CkPrintf("qdcreate called on pe[%d]\n", CkMyPe());
//    fflush(stdout);
        //To compensate for the counter loss for message creation
        //inside the process of immediate message on comm thread
        CkpvAccess(_qd)->create();
    }
#endif
  ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
  proxy->receiveData(msg); // deleted in ProxyPatch::receiveAtoms()
}

recvRegisterProxy()

void ProxyMgr::recvRegisterProxy

(

RegisterProxyMsg *

msg

)

Definition at line 532 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), RegisterProxyMsg::patch, and HomePatch::registerProxy().

                                                 {
  HomePatch *homePatch = PatchMap::Object()->homePatch(msg->patch);
  homePatch->registerProxy(msg); // message deleted in registerProxy()
}

recvResult() [1/2]

void ProxyMgr::recvResult

(

ProxyGBISP1ResultMsg *

msg

)

Definition at line 1857 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), ProxyGBISP1ResultMsg::patch, and HomePatch::receiveResult().

                                                   { //pp -r> hp
  HomePatch *homePatch = PatchMap::Object()->homePatch(msg->patch);
  homePatch->receiveResult(msg); // message deleted in registerProxy()
}

recvResult() [2/2]

void ProxyMgr::recvResult

(

ProxyGBISP2ResultMsg *

msg

)

Definition at line 1872 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), ProxyGBISP2ResultMsg::patch, and HomePatch::receiveResult().

                                                   { //pp -r> hp
  HomePatch *homePatch = PatchMap::Object()->homePatch(msg->patch);
  homePatch->receiveResult(msg); // message deleted in registerProxy()
}

recvResults() [1/3]

void ProxyMgr::recvResults

(

ProxyResultVarsizeMsg *

msg

)

Definition at line 1435 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), ProxyResultVarsizeMsg::patch, and HomePatch::receiveResults().

                                                     {
    HomePatch *home = PatchMap::Object()->homePatch(msg->patch);
    home->receiveResults(msg); // delete done in HomePatch::receiveResults()
}

recvResults() [2/3]

void ProxyMgr::recvResults

(

ProxyResultMsg *

msg

)

Definition at line 1448 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), ProxyResultMsg::patch, and HomePatch::receiveResults().

                                              {
  HomePatch *home = PatchMap::Object()->homePatch(msg->patch);
  home->receiveResults(msg); // delete done in HomePatch::receiveResults()
}

recvResults() [3/3]

void ProxyMgr::recvResults

(

ProxyCombinedResultRawMsg *

omsg

)

Definition at line 1481 of file ProxyMgr.C.

References PatchMap::homePatch(), NAMD_bug(), PatchMap::Object(), ProxyCombinedResultRawMsg::patch, proxyRecvSpanning, and HomePatch::receiveResults().

                                                          {
        ProxyCombinedResultRawMsg *msg = omsg;

//Chao Mei: hack for QD in case of SMP with immediate msg
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
    if(proxyRecvSpanning && msg->isFromImmMsgCall){
//    CkPrintf("qdcreate called on pe[%d]\n", CkMyPe());
//    fflush(stdout);
        //To compensate for the counter loss for message creation
        //inside the process of immediate message on comm thread
        CkpvAccess(_qd)->create();
    }
#endif

  HomePatch *home = PatchMap::Object()->homePatch(msg->patch);
  if (home) {
    //printf("Home got a message\n");
    home->receiveResults(msg); // delete done in HomePatch::receiveResults()
  }
  else {
    NAMD_bug("ProxyMgr should receive result message on home processor");
  }
}

recvSpanningTree()

void ProxyMgr::recvSpanningTree

(

ProxySpanningTreeMsg *

msg

)

Definition at line 1174 of file ProxyMgr.C.

References ResizeArray< Elem >::add(), ResizeArray< Elem >::copy(), ProxySpanningTreeMsg::node, nodecount, PatchMap::Object(), Object(), PatchMap::patch(), ProxySpanningTreeMsg::patch, proxySpanDim, sendSpanningTree(), ProxyPatch::setSpanningTree(), ResizeArray< Elem >::size(), and ProxySpanningTreeMsg::tree.

                                                    {
  int size = msg->tree.size();
  int *child = new int[proxySpanDim];
  int nChild = 0;
  int i;
  ProxyPatch *proxy = (ProxyPatch *) PatchMap::Object()->patch(msg->patch);
  for (i=0; i<proxySpanDim; i++) {
    if (size > i+1) { child[i] = msg->tree[i+1]; nChild++; }
  }
  if (!PatchMap::Object()->homePatch(msg->patch)) {
    proxy->setSpanningTree(msg->node, child, nChild);
  }

  // build subtree and pass down
 if (nChild > 0) {

  nodecount ++;
  //if (nodecount > MAX_INTERNODE) 
  //  iout << "Processor " << CkMyPe() << "has (actual) " << nodecount << " intermediate nodes." << endi;

//CkPrintf("[%d] %d:(%d) %d %d %d %d %d\n", CkMyPe(), msg->patch, size, msg->tree[0], msg->tree[1], msg->tree[2], msg->tree[3], msg->tree[4]);
  NodeIDList *tree = new NodeIDList[proxySpanDim];
  int level = 1, index=1;
  int done = 0;
  while (!done) {
    for (int n=0; n<nChild; n++) {
      if (done) break;
      for (int j=0; j<level; j++) {
       if (index >= size) { done = 1; break; }
       tree[n].add(msg->tree[index]);
       index++;
      }
    }
    level *=proxySpanDim;
  }

  ProxyMgr *proxyMgr = ProxyMgr::Object();
  for (i=0; i<proxySpanDim; i++) {
    if (tree[i].size()) {
      ProxySpanningTreeMsg *cmsg = new ProxySpanningTreeMsg;
      cmsg->patch = msg->patch;
      cmsg->node = CkMyPe();
      cmsg->tree.copy(tree[i]);  // copy data for thread safety
      proxyMgr->sendSpanningTree(cmsg);
    }
  }

  delete [] tree;
 }

  delete [] child;
  delete msg;
}

recvSpanningTreeOnHomePatch()

void ProxyMgr::recvSpanningTreeOnHomePatch	(	int	pid,
		int *	tree,
		int	n
	)

Definition at line 1133 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), and HomePatch::recvSpanningTree().

{
  HomePatch *p = PatchMap::Object()->homePatch(pid);
  p->recvSpanningTree(tree, n);
}

recvUnregisterProxy()

void ProxyMgr::recvUnregisterProxy

(

UnregisterProxyMsg *

msg

)

Definition at line 552 of file ProxyMgr.C.

References PatchMap::homePatch(), PatchMap::Object(), UnregisterProxyMsg::patch, and HomePatch::unregisterProxy().

                                                     {
  HomePatch *homePatch = PatchMap::Object()->homePatch(msg->patch);
  homePatch->unregisterProxy(msg); // message deleted in registerProxy()
}

registerProxy()

void ProxyMgr::registerProxy

(

PatchID

pid

)

Definition at line 518 of file ProxyMgr.C.

References RegisterProxyMsg::node, PatchMap::node(), PatchMap::Object(), and RegisterProxyMsg::patch.

Referenced by ProxyPatch::ProxyPatch().

                                   {
  // determine which node gets message
  NodeID node = PatchMap::Object()->node(pid);

  RegisterProxyMsg *msg = new RegisterProxyMsg;

  msg->node=CkMyPe();
  msg->patch = pid;

  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp[node].recvRegisterProxy(msg);
}

removeProxies()

void ProxyMgr::removeProxies

(

void

)

Definition at line 388 of file ProxyMgr.C.

References UniqueSetIter< T >::begin(), UniqueSet< Elem >::clear(), and UniqueSetIter< T >::end().

Referenced by createProxies(), and ~ProxyMgr().

{
  ProxySetIter pi(proxySet);
  for ( pi = pi.begin(); pi != pi.end(); pi++)
  {
    delete pi->proxyPatch;
  }
  proxySet.clear();
}

removeProxy()

void ProxyMgr::removeProxy

(

PatchID

pid

)

Definition at line 507 of file ProxyMgr.C.

References UniqueSet< Elem >::del(), UniqueSet< Elem >::find(), PatchMap::Object(), ProxyElem::proxyPatch, and PatchMap::unregisterPatch().

Referenced by removeUnusedProxies().

                                 {
  ProxyElem *p = proxySet.find(ProxyElem(pid));
  if (p) { 
    PatchMap::Object()->unregisterPatch(pid,p->proxyPatch);
    delete p->proxyPatch;
    proxySet.del(ProxyElem(pid));
    // iout << iINFO << "Removing unused proxy " << pid << " on " << iPE << ".\n" << endi;
  }
}

removeUnusedProxies()

void ProxyMgr::removeUnusedProxies

(

void

)

Definition at line 398 of file ProxyMgr.C.

References ResizeArray< Elem >::add(), ResizeArray< Elem >::begin(), UniqueSetIter< T >::begin(), ResizeArray< Elem >::end(), UniqueSetIter< T >::end(), and removeProxy().

Referenced by ComputeMgr::updateLocalComputes3().

{
  ResizeArray<PatchID> toDelete;
  ProxySetIter pi(proxySet);
  for ( pi = pi.begin(); pi != pi.end(); pi++)
  {
    if ( pi->proxyPatch->getNumComputes() == 0 ) {
      toDelete.add(pi->patchID);
      //fprintf(stderr, "Proxy Deleted Patch %d Proc %d", pi->patchID, CkMyPe());
    }
  }
  PatchID *pidi = toDelete.begin();
  for ( ; pidi != toDelete.end(); ++pidi ) {
    removeProxy(*pidi);
  }
}

sendNodeAwareSpanningTree()

void ProxyMgr::sendNodeAwareSpanningTree

(

ProxyNodeAwareSpanningTreeMsg *

msg

)

Definition at line 1159 of file ProxyMgr.C.

References ProxyNodeAwareSpanningTreeMsg::allPes, and ProxyNodeAwareSpanningTreeMsg::patch.

Referenced by recvNodeAwareSpanningTree().

                                                                          {
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  int pe = msg->allPes[0]; //the root procID

#if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
  DebugFileTrace *dft = DebugFileTrace::Object();
  dft->openTrace();
  dft->writeTrace("PMgr::sndST: from proc %d for patch[%d]\n", pe, msg->patch);
  dft->closeTrace();
#endif

  cp[pe].recvNodeAwareSpanningTree(msg);
}

sendNodeAwareSpanningTreeToHomePatch()

void ProxyMgr::sendNodeAwareSpanningTreeToHomePatch	(	int	pid,
		proxyTreeNode *	tree,
		int	n
	)

Definition at line 1139 of file ProxyMgr.C.

References ProxyNodeAwareSpanningTreeMsg::getANewMsg(), PatchMap::node(), and PatchMap::Object().

Referenced by sendSpanningTrees().

{
  CProxy_ProxyMgr cp(thisgroup);
  ProxyNodeAwareSpanningTreeMsg *msg = ProxyNodeAwareSpanningTreeMsg::getANewMsg(pid, -1, tree, n);
  cp[PatchMap::Object()->node(pid)].recvNodeAwareSpanningTreeOnHomePatch(msg);
}

sendProxies()

void ProxyMgr::sendProxies	(	int	pid,
		int *	list,
		int	n
	)

Definition at line 599 of file ProxyMgr.C.

Referenced by HomePatch::sendProxies().

{
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp[0].recvProxies(pid, list, n);
}

sendProxyAll()

void ProxyMgr::sendProxyAll	(	ProxyDataMsg *	msg,
		int	pcnt,
		int *	pids
	)

Definition at line 1676 of file ProxyMgr.C.

References proxySendSpanning.

Referenced by HomePatch::positionsReady(), HomePatch::positionsReady_SOA(), and recvImmediateProxyAll().

                                                             {
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
    if(proxySendSpanning == 1) {
        CProxy_NodeProxyMgr cnp(CkpvAccess(BOCclass_group).nodeProxyMgr);
        for(int i=0; i<pcnt-1; i++) {
            ProxyDataMsg *copymsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);
            cnp[pids[i]].recvImmediateProxyAll(copymsg);
        }
        cnp[pids[pcnt-1]].recvImmediateProxyAll(msg);
        return;
    }
#endif
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp.recvImmediateProxyAll(msg,pcnt,pids);
}

sendProxyData()

void ProxyMgr::sendProxyData	(	ProxyDataMsg *	msg,
		int	pcnt,
		int *	pids
	)

Definition at line 1562 of file ProxyMgr.C.

References proxySendSpanning.

Referenced by HomePatch::positionsReady(), HomePatch::positionsReady_SOA(), and recvImmediateProxyData().

                                                              {
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
    if(proxySendSpanning == 1) {
        CProxy_NodeProxyMgr cnp(CkpvAccess(BOCclass_group).nodeProxyMgr);
        for(int i=0; i<pcnt-1; i++) {
            ProxyDataMsg *copymsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);
            cnp[pids[i]].recvImmediateProxyData(copymsg);
        }
        cnp[pids[pcnt-1]].recvImmediateProxyData(msg);
        return;
    }
#endif
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp.recvImmediateProxyData(msg,pcnt,pids);
}

sendResult() [1/2]

void ProxyMgr::sendResult

(

ProxyGBISP1ResultMsg *

msg

)

Definition at line 1850 of file ProxyMgr.C.

References PatchMap::node(), PatchMap::Object(), and ProxyGBISP1ResultMsg::patch.

Referenced by ProxyPatch::boxClosed().

                                                   { //pp -r> hp
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  NodeID node = PatchMap::Object()->node(msg->patch);
  CmiEnableUrgentSend(1);
  cp[node].recvResult(msg);
  CmiEnableUrgentSend(0);
}

sendResult() [2/2]

void ProxyMgr::sendResult

(

ProxyGBISP2ResultMsg *

msg

)

Definition at line 1865 of file ProxyMgr.C.

References PatchMap::node(), PatchMap::Object(), and ProxyGBISP2ResultMsg::patch.

                                                   { //pp -r> hp
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  NodeID node = PatchMap::Object()->node(msg->patch);
  CmiEnableUrgentSend(1);
  cp[node].recvResult(msg);
  CmiEnableUrgentSend(0);
}

sendResults() [1/3]

void ProxyMgr::sendResults

(

ProxyResultVarsizeMsg *

msg

)

Definition at line 1427 of file ProxyMgr.C.

References PatchMap::node(), PatchMap::Object(), and ProxyResultVarsizeMsg::patch.

                                                     {
    CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
    NodeID node = PatchMap::Object()->node(msg->patch);
    CmiEnableUrgentSend(1);
    cp[node].recvResults(msg);
    CmiEnableUrgentSend(0);
}

sendResults() [2/3]

void ProxyMgr::sendResults

(

ProxyResultMsg *

msg

)

Definition at line 1440 of file ProxyMgr.C.

References PatchMap::node(), PatchMap::Object(), and ProxyResultMsg::patch.

                                              {
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  NodeID node = PatchMap::Object()->node(msg->patch);
  CmiEnableUrgentSend(1);
  cp[node].recvResults(msg);
  CmiEnableUrgentSend(0);
}

sendResults() [3/3]

void ProxyMgr::sendResults

(

ProxyCombinedResultMsg *

msg

)

Definition at line 1454 of file ProxyMgr.C.

References ProxyPatch::depositCombinedResultMsg(), ProxyPatch::getSpanningTreeParent(), PatchMap::Object(), PatchMap::patch(), ProxyCombinedResultMsg::patch, and ProxyCombinedResultMsg::toRaw().

                                                      {
  ProxyPatch *patch = (ProxyPatch *)PatchMap::Object()->patch(msg->patch);
  ProxyCombinedResultMsg *ocMsg = patch->depositCombinedResultMsg(msg);
  if (ocMsg) {
        ProxyCombinedResultRawMsg *cMsg = ProxyCombinedResultMsg::toRaw(ocMsg);        
    int destPe = patch->getSpanningTreeParent();
    CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
    CmiAssert(destPe!=CkMyPe());
    //if(destPe != CkMyPe()) {
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
      /*CkPrintf("ready to call node::recvImmRes on pe[%d] to dest[%d]\n", CkMyPe(), destPe);
      fflush(stdout);*/

      cMsg->destPe = destPe;
      CProxy_NodeProxyMgr cnp(CkpvAccess(BOCclass_group).nodeProxyMgr);
      cnp[CkNodeOf(destPe)].recvImmediateResults(cMsg);
#else
      cp[destPe].recvImmediateResults(cMsg);
#endif
    //}
    //else{
    //  cp[destPe].recvResults(cMsg);
    //}
  }
}

sendSpanningTree()

void ProxyMgr::sendSpanningTree

(

ProxySpanningTreeMsg *

msg

)

Definition at line 1154 of file ProxyMgr.C.

References ProxySpanningTreeMsg::tree.

Referenced by recvSpanningTree(), and HomePatch::sendSpanningTree().

                                                    {
  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp[msg->tree[0]].recvSpanningTree(msg);
}

sendSpanningTrees()

void ProxyMgr::sendSpanningTrees

(

)

Definition at line 1106 of file ProxyMgr.C.

References ResizeArray< Elem >::begin(), PatchMap::numPatches(), numProxies(), PatchMap::Object(), Object(), ProxyTree::proxylist, sendNodeAwareSpanningTreeToHomePatch(), sendSpanningTreeToHomePatch(), ResizeArray< Elem >::size(), and ProxyTree::trees.

Referenced by recvPatchProxyInfo(), recvProxies(), and ComputeMgr::updateLocalComputes5().

{
  int numPatches = PatchMap::Object()->numPatches();
  for (int pid=0; pid<numPatches; pid++) {
    int numProxies = ptree.proxylist[pid].size();
#ifdef NODEAWARE_PROXY_SPANNINGTREE
    if (numProxies == 0)
      ProxyMgr::Object()->sendNodeAwareSpanningTreeToHomePatch(pid, NULL, 0);
    else {
      ProxyMgr::Object()->sendNodeAwareSpanningTreeToHomePatch(pid, ptree.naTrees[pid].begin(), ptree.naTrees[pid].size());
    }
#else
    if (numProxies == 0)
      ProxyMgr::Object()->sendSpanningTreeToHomePatch(pid, NULL, 0);
    else {
      ProxyMgr::Object()->sendSpanningTreeToHomePatch(pid, ptree.trees[pid].begin(), ptree.trees[pid].size());
    }
#endif
  }
}

sendSpanningTreeToHomePatch()

void ProxyMgr::sendSpanningTreeToHomePatch	(	int	pid,
		int *	tree,
		int	n
	)

Definition at line 1127 of file ProxyMgr.C.

References PatchMap::node(), and PatchMap::Object().

Referenced by sendSpanningTrees().

{
  CProxy_ProxyMgr cp(thisgroup);
  cp[PatchMap::Object()->node(pid)].recvSpanningTreeOnHomePatch(pid, tree, n);
}

setProxyTreeBranchFactor()

void ProxyMgr::setProxyTreeBranchFactor

(

int

dim

)

Definition at line 379 of file ProxyMgr.C.

References proxySpanDim.

Referenced by Node::startup().

                                              {
    if(CkMyRank()!=0) return;
    proxySpanDim = dim;
}

setRecvSpanning()

void ProxyMgr::setRecvSpanning

(

)

Definition at line 370 of file ProxyMgr.C.

References proxyRecvSpanning.

Referenced by WorkDistrib::savePatchMap(), WorkDistrib::sendPatchMap(), and Node::startup().

                               {
  if(CkMyRank()!=0) return;
  proxyRecvSpanning = 1;
}

setSendSpanning()

void ProxyMgr::setSendSpanning

(

)

Definition at line 361 of file ProxyMgr.C.

References proxySendSpanning.

Referenced by WorkDistrib::savePatchMap(), WorkDistrib::sendPatchMap(), and Node::startup().

                               {
  if(CkMyRank()!=0) return; 
  proxySendSpanning = 1;
}

unregisterProxy()

void ProxyMgr::unregisterProxy

(

PatchID

pid

)

Definition at line 538 of file ProxyMgr.C.

References UnregisterProxyMsg::node, PatchMap::node(), PatchMap::Object(), and UnregisterProxyMsg::patch.

Referenced by ProxyPatch::~ProxyPatch().

                                     {
  // determine which node gets message
  NodeID node = PatchMap::Object()->node(pid);

  UnregisterProxyMsg *msg = new UnregisterProxyMsg;

  msg->node=CkMyPe();
  msg->patch = pid;

  CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
  cp[node].recvUnregisterProxy(msg);
}

Member Data Documentation

nodecount

int ProxyMgr::nodecount = 0

static

Definition at line 398 of file ProxyMgr.h.

Referenced by recvNodeAwareSpanningTree(), recvSpanningTree(), and ComputeMgr::updateLocalComputes3().

---

The documentation for this class was generated from the following files:

• ProxyMgr.h
• ProxyMgr.C