NodeProxyMgr Class Reference

#include <ProxyMgr.h>

Inheritance diagram for NodeProxyMgr:

Public Member Functions
	NodeProxyMgr ()
	~NodeProxyMgr ()
void	createProxyInfo (int numPs)
void	registerPatch (int patchID, int numPes, int *pes)
proxyTreeNode *	getPatchProxyInfo (int patchID)
void	registerLocalProxyMgr (CkGroupID one)
const CkGroupID &	getLocalProxyMgr ()
void	registerLocalPatchMap (int rank, PatchMap *one)
PatchMap *	getLocalPatchMap (int rank)
void	recvImmediateProxyData (ProxyDataMsg *msg)
void	recvImmediateProxyAll (ProxyDataMsg *msg)
void	recvImmediateResults (ProxyCombinedResultRawMsg *)
void	createSTForHomePatches (PatchMap *pmap)
void	sendProxyList (int pid, int *plist, int size)
void	sendProxyListInfo (PatchProxyListMsg *msg)
void	contributeToParent ()

Detailed Description

Definition at line 426 of file ProxyMgr.h.

Constructor & Destructor Documentation

NodeProxyMgr()

NodeProxyMgr::NodeProxyMgr ( )

inline

Definition at line 450 of file ProxyMgr.h.

                  {
        proxyInfo = NULL;
        numPatches = 0;
        localPatchMaps = new PatchMap *[CkMyNodeSize()];

                parentNode = -1;
                numKidNodes = 0;
                kidRecved = 0;
                numHomePatches = 0;
                homepatchRecved = 0;
                localProxyLists = NULL;
                remoteProxyLists = NULL;
                localDepositLock = CmiCreateLock();
                remoteDepositLock = CmiCreateLock();
    }

~NodeProxyMgr()

NodeProxyMgr::~NodeProxyMgr ( )

inline

Definition at line 465 of file ProxyMgr.h.

                   {
        for(int i=0; i<numPatches; i++) {
            delete proxyInfo[i];
        }
        delete [] proxyInfo;
        delete [] localPatchMaps;

                CmiDestroyLock(localDepositLock);
                CmiDestroyLock(remoteDepositLock);
    }

Member Function Documentation

contributeToParent()

void NodeProxyMgr::contributeToParent

(

)

Definition at line 2007 of file ProxyMgr.C.

References PatchProxyListMsg::createPatchProxyListMsg().

Referenced by sendProxyList(), and sendProxyListInfo().

                                     {
        if(homepatchRecved!=numHomePatches || kidRecved != numKidNodes) return;

        homepatchRecved = 0;
        kidRecved = 0;
        //construct the msg
        PatchProxyListMsg *msg = PatchProxyListMsg::createPatchProxyListMsg(remoteProxyLists, numKidNodes, localProxyLists, numHomePatches);
        if(parentNode == -1) {
                //send to proxy mgr on PE[0] as this is the root node
                CProxy_ProxyMgr cp(CkpvAccess(BOCclass_group).proxyMgr);
                cp[0].recvPatchProxyInfo(msg);
        }else{
                CProxy_NodeProxyMgr cnp(thisgroup);
                cnp[parentNode].sendProxyListInfo(msg);
        }
        for(int i=0; i<numKidNodes; i++) {
                delete remoteProxyLists[i];
        }
}

createProxyInfo()

void NodeProxyMgr::createProxyInfo ( int  numPs )

inline

Definition at line 476 of file ProxyMgr.h.

                                   {
        numPatches = numPs;
        proxyInfo = new proxyTreeNode *[numPs];
        memset(proxyInfo, 0, sizeof(proxyTreeNode *)*numPs);
    }

createSTForHomePatches()

void NodeProxyMgr::createSTForHomePatches

(

PatchMap *

pmap

)

Definition at line 1921 of file ProxyMgr.C.

References HOMEPATCH_TREE_BRFACTOR, and PatchMap::numPatchesOnNode().

                                                       {
        //We use implicit tree construction for all home patches
        std::vector<int> nodesWithPatches; //record the id of node that has home patches
        int myNodeIdx = -1; //the index into the above vector of this node
        for(int nodeId=0; nodeId<CkNumNodes(); ++nodeId) {
                int hpCnt = 0;
                int firstPe = CkNodeFirst(nodeId);
                int endPe = firstPe + CkNodeSize(nodeId);
                for(int pe=firstPe; pe < endPe; ++pe) {
                        hpCnt += pmap->numPatchesOnNode(pe);
                }
                if(hpCnt==0) continue;

                nodesWithPatches.push_back(nodeId);
                if(CkMyNode() == nodeId) {
                        //on my node
                        myNodeIdx = nodesWithPatches.size()-1;
                        numHomePatches = hpCnt;
                        homepatchRecved = 0;
                        localProxyLists = new ProxyListInfo[hpCnt];
                        memset(localProxyLists, 0, sizeof(ProxyListInfo)*hpCnt);                        
                }
        }

        if(myNodeIdx==-1){
                //there's no home patches on this node
                //just set to a value that doesn't make sense in spanning tree.
                parentNode = -2; 
                numKidNodes = 0;
                kidRecved = 0;
                return;
        }
        
        //calculate parent
        if(myNodeIdx == 0) {
                parentNode = -1;
        }else{
                int parentIdx = (myNodeIdx-1)/HOMEPATCH_TREE_BRFACTOR;
                parentNode = nodesWithPatches[parentIdx];
        }

        //calculate kids
        numKidNodes = 0;
        int totalNodes = nodesWithPatches.size();
        for(int i=1; i<=HOMEPATCH_TREE_BRFACTOR; i++) {
                int kidId = myNodeIdx*HOMEPATCH_TREE_BRFACTOR+i;
                if(kidId >= totalNodes) break;
                numKidNodes++;
        }
        if(numKidNodes!=0) {
                remoteProxyLists = new PatchProxyListMsg *[numKidNodes];
        }
        kidRecved = 0;

        //CkPrintf("Node[%d] has %d homepatches with parent=%d and %d kids \n", CkMyNode(), numHomePatches, parentNode, numKidNodes);
}

getLocalPatchMap()

PatchMap* NodeProxyMgr::getLocalPatchMap ( int  rank )

inline

Definition at line 495 of file ProxyMgr.h.

                                        {
        return localPatchMaps[rank];
    }   

getLocalProxyMgr()

const CkGroupID& NodeProxyMgr::getLocalProxyMgr ( )

inline

Definition at line 489 of file ProxyMgr.h.

                                       {
        return localProxyMgr;
    }

getPatchProxyInfo()

proxyTreeNode* NodeProxyMgr::getPatchProxyInfo ( int  patchID )

inline

Definition at line 482 of file ProxyMgr.h.

                                                 {
        return proxyInfo[patchID];
    }

recvImmediateProxyAll()

void NodeProxyMgr::recvImmediateProxyAll

(

ProxyDataMsg *

msg

)

Definition at line 1745 of file ProxyMgr.C.

References proxyTreeNode::numPes, ProxyDataMsg::patch, PatchMap::patch(), and proxyTreeNode::peIDs.

                                                          {    
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
    CProxy_ProxyMgr cp(localProxyMgr);
    proxyTreeNode *ptn = proxyInfo[msg->patch];
    CmiAssert(ptn->numPes!=0);
    #if defined(PROCTRACE_DEBUG) && defined(NAST_DEBUG)
    //This could be executed on comm thd.
    printf("NodePMgr::recvImmPAll for patch[%d] on node %d rank %d, prepare to send proc ", msg->patch, CkMyNode(), CkMyRank());
    for(int i=0; i<ptn->numPes; i++) {
        printf("%d, ", ptn->peIDs[i]);
    }
    printf("\n");
    fflush(stdout);
    #endif

    //re-send msg to this nodes's children nodes.
    //only the first pe of a node of node-aware ST should contain children nodes
    int rank = CkRankOf(ptn->peIDs[0]);
    PatchMap *pmap = localPatchMaps[rank];
    ProxyPatch *ppatch = (ProxyPatch *)pmap->patch(msg->patch);

    int npid = ppatch->getSTNNodeChild();
    int *pids = ppatch->getSTNodeChildPtr();
    if(npid>0) {        
        //only needs to send to other nodes, so check the last entry of pids.
        //This is because the data for proxies on the same node have been sent
        //later in this function by NodeProxyMgr.
        if(pids[npid-1]==CkMyNode()) npid--;
    }
    
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
    if (npid) {
        int ntreephs;
        PersistentHandle *treephs = ppatch->getSpanningTreePhs(ntreephs);
        CmiAssert(treephs && ntreephs >= npid);
        CmiUsePersistentHandle(treephs, ntreephs);
    }
#endif
    CProxy_NodeProxyMgr cnp(thisgroup);
    for(int i=0; i<npid; i++) {
        ProxyDataMsg *copymsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);
        cnp[pids[i]].recvImmediateProxyAll(copymsg);
    }    
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
    CmiUsePersistentHandle(NULL, 0);
#endif

    //re-send msg to it's internal cores
#if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
    msg->isFromImmMsgCall = (CkMyRank()==CkMyNodeSize());
#endif
    cp.recvProxyAll(msg, ptn->numPes, ptn->peIDs);
#else
    CkAbort("Bad execution path to NodeProxyMgr::recvImmediateProxyData\n");
#endif
}

recvImmediateProxyData()

void NodeProxyMgr::recvImmediateProxyData

(

ProxyDataMsg *

msg

)

Definition at line 1628 of file ProxyMgr.C.

References proxyTreeNode::numPes, ProxyDataMsg::patch, PatchMap::patch(), and proxyTreeNode::peIDs.

                                                           {    
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
    CProxy_ProxyMgr cp(localProxyMgr);
    proxyTreeNode *ptn = proxyInfo[msg->patch];
    CmiAssert(ptn->numPes!=0);

    //re-send msg to this nodes's children nodes.
    //only the first pe of a node of node-aware ST should contain children nodes
    int rank = CkRankOf(ptn->peIDs[0]);
    PatchMap *pmap = localPatchMaps[rank];
    ProxyPatch *ppatch = (ProxyPatch *)pmap->patch(msg->patch);

    int npid = ppatch->getSTNNodeChild();
    int *pids = ppatch->getSTNodeChildPtr();
    if(npid>0) {        
        //only needs to send to other nodes, so check the last entry of pids.
        //This is because the data for proxies on the same node have been sent
        //later in this function by NodeProxyMgr.
        if(pids[npid-1]==CkMyNode()) npid--;
    }    
    CProxy_NodeProxyMgr cnp(thisgroup);
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
    if (npid) {
        int ntreephs;
        PersistentHandle *treephs = ppatch->getSpanningTreePhs(ntreephs);
        CmiAssert(treephs && ntreephs >= npid);
        CmiUsePersistentHandle(treephs, ntreephs);
    }
#endif
    for(int i=0; i<npid; i++) {
        ProxyDataMsg *copymsg = (ProxyDataMsg *)CkCopyMsg((void **)&msg);
        cnp[pids[i]].recvImmediateProxyData(copymsg);
    }    
#if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE
    CmiUsePersistentHandle(NULL, 0);
#endif

    //re-send msg to it's internal cores
#if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
    msg->isFromImmMsgCall = (CkMyRank()==CkMyNodeSize());
#endif
    cp.recvProxyData(msg, ptn->numPes, ptn->peIDs);
#else
    CkAbort("Bad execution path to NodeProxyMgr::recvImmediateProxyData\n");
#endif
}

recvImmediateResults()

void NodeProxyMgr::recvImmediateResults

(

ProxyCombinedResultRawMsg *

omsg

)

Definition at line 1524 of file ProxyMgr.C.

References ProxyPatch::depositCombinedResultRawMsg(), ProxyPatch::getSpanningTreeParent(), PatchMap::homePatch(), PatchMap::patch(), ProxyCombinedResultRawMsg::patch, and ProxyCombinedResultMsg::toRaw().

                                                                      {
    ProxyCombinedResultRawMsg *msg = omsg;
#if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
    //CkPrintf("recvImmRes called on comm thread%d pe[%d]\n", CkMyRank()==CmiMyNodeSize(), CkMyPe());
    //fflush(stdout);

    int destRank = CkRankOf(msg->destPe);
    PatchMap *pmap = localPatchMaps[destRank];
    HomePatch *home = pmap->homePatch(msg->patch);
    if (home) {
#if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
        msg->isFromImmMsgCall = (CkMyRank()==CkMyNodeSize());
#endif
        CProxy_ProxyMgr cp(localProxyMgr);
        CmiEnableUrgentSend(1);
        cp[msg->destPe].recvResults(msg);
        CmiEnableUrgentSend(0);
/*
        char *srcfrom = "Isfrom";
        if(CkMyRank()!=CmiMyNodeSize()) srcfrom="Notfrom";
      CkPrintf("%s comm thread from pe[%d]\n", srcfrom, CkMyPe());
      fflush(stdout);
*/
    }
    else {
        ProxyPatch *patch = (ProxyPatch *)pmap->patch(msg->patch);
        ProxyCombinedResultMsg *ocMsg = patch->depositCombinedResultRawMsg(msg); 
        if (ocMsg) {
            CProxy_NodeProxyMgr cnp(thisgroup);
            ocMsg->destPe = patch->getSpanningTreeParent();
                        ProxyCombinedResultRawMsg *cMsg = ProxyCombinedResultMsg::toRaw(ocMsg);
            cnp[CkNodeOf(cMsg->destPe)].recvImmediateResults(cMsg);
        }
    }
#endif
}

registerLocalPatchMap()

void NodeProxyMgr::registerLocalPatchMap ( int  rank, PatchMap *  one  )

inline

Definition at line 492 of file ProxyMgr.h.

                                                       {
        localPatchMaps[rank] = one;
    }

registerLocalProxyMgr()

void NodeProxyMgr::registerLocalProxyMgr ( CkGroupID  one )

inline

Definition at line 486 of file ProxyMgr.h.

                                             {
        localProxyMgr = one;
    }

registerPatch()

void NodeProxyMgr::registerPatch	(	int	patchID,
		int	numPes,
		int *	pes
	)

Definition at line 1839 of file ProxyMgr.C.

Referenced by ProxyMgr::recvNodeAwareSpanningTree().

                                                                 {
    if(proxyInfo[patchID]) {
        delete proxyInfo[patchID];
    }
    if(numPes == 0) {
        proxyInfo[patchID] = NULL;
    }else{
        proxyInfo[patchID] = new proxyTreeNode(CkNodeOf(pes[0]),numPes,pes);
    }
}

sendProxyList()

void NodeProxyMgr::sendProxyList	(	int	pid,
		int *	plist,
		int	size
	)

Definition at line 1978 of file ProxyMgr.C.

References contributeToParent(), ProxyListInfo::numProxies, ProxyListInfo::patchID, and ProxyListInfo::proxyList.

Referenced by HomePatch::sendProxies().

                                                             {
        int insertIdx; //indexed from 0
        CmiLock(localDepositLock);
        insertIdx = homepatchRecved++; //ensure the atomic increment

        localProxyLists[insertIdx].patchID = pid;
        localProxyLists[insertIdx].numProxies = size;
        localProxyLists[insertIdx].proxyList = plist;

        if(insertIdx == (numHomePatches-1)) {
                //all local home patches have contributed
                contributeToParent();
        }
        CmiUnlock(localDepositLock);
}

sendProxyListInfo()

void NodeProxyMgr::sendProxyListInfo

(

PatchProxyListMsg *

msg

)

Definition at line 1994 of file ProxyMgr.C.

References contributeToParent().

                                                          {
        int insertIdx; //indexed from 0
        CmiLock(localDepositLock);
        insertIdx = kidRecved++;
        
        remoteProxyLists[insertIdx] = msg;
        if(insertIdx == (numKidNodes-1)) {
                //all kids have contributed;
                contributeToParent();
        }
        CmiUnlock(localDepositLock);
}

---

The documentation for this class was generated from the following files:

• ProxyMgr.h
• ProxyMgr.C