ComputeMgr Class Reference

#include <ComputeMgr.h>

Inheritance diagram for ComputeMgr:

Public Member Functions
	ComputeMgr ()
	~ComputeMgr ()
void	createComputes (ComputeMap *map)
void	updateComputes (int, CkGroupID)
void	updateComputes2 (CkQdMsg *)
void	updateComputes3 ()
void	splitComputes ()
void	splitComputes2 (CkQdMsg *)
void	updateLocalComputes ()
void	updateLocalComputes2 (CkQdMsg *)
void	updateLocalComputes3 ()
void	updateLocalComputes4 (CkQdMsg *)
void	updateLocalComputes5 ()
void	doneUpdateLocalComputes ()
void	sendComputeGlobalConfig (ComputeGlobalConfigMsg *)
void	recvComputeGlobalConfig (ComputeGlobalConfigMsg *)
void	sendComputeGlobalData (ComputeGlobalDataMsg *)
void	recvComputeGlobalData (ComputeGlobalDataMsg *)
void	sendComputeGlobalResults (ComputeGlobalResultsMsg *)
void	recvComputeGlobalResults (ComputeGlobalResultsMsg *)
void	enableComputeGlobalResults ()
void	sendComputeDPMEData (ComputeDPMEDataMsg *)
void	recvComputeDPMEData (ComputeDPMEDataMsg *)
void	sendComputeDPMEResults (ComputeDPMEResultsMsg *, int)
void	recvComputeDPMEResults (ComputeDPMEResultsMsg *)
void	sendComputeEwaldData (ComputeEwaldMsg *)
void	recvComputeEwaldData (ComputeEwaldMsg *)
void	sendComputeEwaldResults (ComputeEwaldMsg *)
void	recvComputeEwaldResults (ComputeEwaldMsg *)
void	recvComputeConsForceMsg (ComputeConsForceMsg *)
void	recvCudaGlobalMasterCreateMsg (std::vector< std::string > args)
void	recvCudaGlobalMasterRemoveMsg (std::vector< std::string > args)
void	recvCudaGlobalMasterUpdateMsg (std::vector< std::string > args)
void	recvCudaGlobalMasterUpdateResultMsg (int tcl_error_code, std::vector< std::string > args)
std::string	getCudaGlobalMasterUpdateResultString (const std::string &client_name) const
int	getCudaGlobalMasterUpdateResult (const std::string &client_name) const
void	sendYieldDevice (int pe)
void	recvYieldDevice (int pe)
void	sendAssignPatchesOnPe (std::vector< int > &pes, CudaComputeNonbonded *c)
void	recvAssignPatchesOnPe (CudaComputeNonbondedMsg *msg)
void	sendSkipPatchesOnPe (std::vector< int > &pes, CudaComputeNonbonded *c)
void	recvSkipPatchesOnPe (CudaComputeNonbondedMsg *msg)
void	sendFinishPatchesOnPe (std::vector< int > &pes, CudaComputeNonbonded *c)
void	recvFinishPatchesOnPe (CudaComputeNonbondedMsg *msg)
void	sendFinishPatchOnPe (int pe, CudaComputeNonbonded *c, int i, PatchID patchID)
void	recvFinishPatchOnPe (CudaComputeNonbondedMsg *msg)
void	sendOpenBoxesOnPe (std::vector< int > &pes, CudaComputeNonbonded *c)
void	recvOpenBoxesOnPe (CudaComputeNonbondedMsg *msg)
void	sendFinishReductions (int pe, CudaComputeNonbonded *c)
void	recvFinishReductions (CudaComputeNonbondedMsg *msg)
void	sendMessageEnqueueWork (int pe, CudaComputeNonbonded *c)
void	recvMessageEnqueueWork (CudaComputeNonbondedMsg *msg)
void	sendLaunchWork (int pe, CudaComputeNonbonded *c)
void	recvLaunchWork (CudaComputeNonbondedMsg *msg)
void	sendUnregisterBoxesOnPe (std::vector< int > &pes, CudaComputeNonbonded *c)
void	recvUnregisterBoxesOnPe (CudaComputeNonbondedMsg *msg)

Public Attributes
ComputeGlobal *	computeGlobalObject
ResizeArray< ComputeGlobalResultsMsg * >	computeGlobalResultsMsgs
int	computeGlobalResultsMsgSeq
int	computeGlobalResultsMsgMasterSeq
CkCallback	callMeBackCB

Detailed Description

Definition at line 66 of file ComputeMgr.h.

Constructor & Destructor Documentation

ComputeMgr()

ComputeMgr::ComputeMgr

(

)

Definition at line 117 of file ComputeMgr.C.

References computeGlobalObject, computeGlobalResultsMsgMasterSeq, and computeGlobalResultsMsgSeq.

{
    CkpvAccess(BOCclass_group).computeMgr = thisgroup;
    computeGlobalObject = 0;
    computeGlobalResultsMsgSeq = -1;
    computeGlobalResultsMsgMasterSeq = -1;
    computeDPMEObject = 0;
    computeEwaldObject = 0;
    computeNonbondedWorkArrays = new ComputeNonbondedWorkArrays;
    skipSplitting = 0;
    masterServerObject = NULL;
}

~ComputeMgr()

ComputeMgr::~ComputeMgr

(

void

)

Definition at line 130 of file ComputeMgr.C.

References endi(), iINFO(), and iout.

{
    delete computeNonbondedWorkArrays;
    if (masterServerObject != NULL) delete masterServerObject;
    for (auto& loader: CudaGlobalMasterClientDlloaders) {
      if (loader) {
        iout << iINFO << "Close library " << loader->LibName() << "\n" << endi;
        loader->DLCloseLib();
      }
    }
}

Member Function Documentation

createComputes()

void ComputeMgr::createComputes

(

ComputeMap *

map

)

Definition at line 1037 of file ComputeMgr.C.

References GlobalMasterServer::addClient(), CudaComputeNonbonded::assignPatches(), computeAnglesType, computeAnisoType, computeBondsType, computeCrosstermsType, computeDihedralsType, computeExclsType, computeImpropersType, computeNonbondedCUDA2Type, computeNonbondedPairType, computeNonbondedSelfType, computeOneFourNbTholeType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfImpropersType, computeSelfOneFourNbTholeType, computeSelfTholeType, computeTholeType, DebugM, DeviceCUDA::device_shared_with_pe(), deviceCUDA, getCudaComputeNonbonded(), DeviceCUDA::getMasterPe(), CudaComputeNonbonded::initialize(), Node::molecule, Node::myid(), NAMD_BONDEDGPU_ANGLES, NAMD_BONDEDGPU_ANISOS, NAMD_BONDEDGPU_BONDS, NAMD_BONDEDGPU_CROSSTERMS, NAMD_BONDEDGPU_DIHEDRALS, NAMD_BONDEDGPU_EXCLS, NAMD_BONDEDGPU_IMPROPERS, NAMD_BONDEDGPU_ONEFOURENBTHOLES, NAMD_BONDEDGPU_THOLES, Molecule::numAtoms, PatchMap::Object(), Node::Object(), Node::simParameters, simParams, and ComputeMap::type().

Referenced by Node::startup().

{
// #ifdef NAMD_CUDA
//     int ComputePmeCUDACounter = 0;
// #endif
    Node *node = Node::Object();
    SimParameters *simParams = node->simParameters;
    int myNode = node->myid();

    if ( simParams->globalForcesOn && !myNode )
    {
        DebugM(4,"Mgr running on Node "<<CkMyPe()<<"\n");
        /* create a master server to allow multiple masters */
        masterServerObject = new GlobalMasterServer(this,
                PatchMap::Object()->numNodesWithPatches());

        #ifdef NODEGROUP_FORCE_REGISTER
        CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
        PatchData *patchData = cpdata.ckLocalBranch();
        patchData->master_mgr = this;
        #endif

        /* create the individual global masters */
        // masterServerObject->addClient(new GlobalMasterTest());
        if (simParams->tclForcesOn)
            masterServerObject->addClient(new GlobalMasterTcl());
        if (simParams->IMDon && ! (simParams->IMDignore || simParams->IMDignoreForces) )
            masterServerObject->addClient(new GlobalMasterIMD());
        // SMD is implemented on GPU resident version of NAMD (NAMD3)
        if (simParams->SMDOn && !simParams->CUDASOAintegrateMode)
            masterServerObject->addClient(
                new GlobalMasterSMD(simParams->SMDk, simParams->SMDk2,
                                    simParams->SMDVel,
                                    simParams->SMDDir, simParams->SMDOutputFreq,
                                    simParams->firstTimestep, simParams->SMDFile,
                                    node->molecule->numAtoms)
            );
            
        if (simParams->symmetryOn && 
          (simParams->firstTimestep < simParams->symmetryLastStep || 
          simParams->symmetryLastStep == -1))
            masterServerObject->addClient(new GlobalMasterSymmetry());    
        if (simParams->TMDOn)
            masterServerObject->addClient(new GlobalMasterTMD());
        if (simParams->miscForcesOn)
            masterServerObject->addClient(new GlobalMasterMisc());
        if ( simParams->freeEnergyOn )
            masterServerObject->addClient(new GlobalMasterFreeEnergy());
        if ( simParams->colvarsOn )
            masterServerObject->addClient(new GlobalMasterColvars());

    }

    if ( !myNode && simParams->IMDon && (simParams->IMDignore || simParams->IMDignoreForces) ) {
      // GlobalMasterIMD constructor saves pointer to node->IMDOutput object
      new GlobalMasterIMD();
    }

#if defined(NAMD_CUDA) || defined(NAMD_HIP)
    bool deviceIsMine = ( deviceCUDA->getMasterPe() == CkMyPe() );
#ifdef BONDED_CUDA
    // Place bonded forces on Pe different from non-bonded forces
    int bondedMasterPe = deviceCUDA->getMasterPe();
    // for (int i=0;i < deviceCUDA->getNumPesSharingDevice();i++) {
    //   int pe = deviceCUDA->getPesSharingDevice(i);
    //   if (pe != deviceCUDA->getMasterPe()) {
    //     bondedMasterPe = pe;
    //   }
    // }
    bool deviceIsMineBonded = (CkMyPe() == bondedMasterPe);
#endif
#endif

    for (int i=0; i < map->nComputes; i++)
    {
        if ( ! ( i % 100 ) )
        {
        }

#if defined(NAMD_CUDA) || defined(NAMD_HIP)
        switch ( map->type(i) )
        {
          // case computePmeCUDAType:
          //   // Only create single ComputePmeCUDA object per Pe
          //  if ( map->computeData[i].node != myNode ) continue;
          //  if (ComputePmeCUDACounter > 0) continue;
          //  ComputePmeCUDACounter++;
          //  break;
          case computeNonbondedSelfType:
            if ( ! deviceIsMine ) continue;
            if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
          break;

          case computeNonbondedPairType:
            if ( ! deviceIsMine ) continue;
            if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
          break;

#ifdef BONDED_CUDA
          case computeSelfBondsType:
          case computeBondsType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_BONDS) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfAnglesType:
          case computeAnglesType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_ANGLES) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfDihedralsType:
          case computeDihedralsType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_DIHEDRALS) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfImpropersType:
          case computeImpropersType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_IMPROPERS) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfExclsType:
          case computeExclsType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_EXCLS) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfCrosstermsType:
          case computeCrosstermsType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_CROSSTERMS) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfTholeType:
          case computeTholeType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_THOLES) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfAnisoType:
          case computeAnisoType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_ANISOS) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeSelfOneFourNbTholeType:
          case computeOneFourNbTholeType:
            if (simParams->bondedCUDA & NAMD_BONDEDGPU_ONEFOURENBTHOLES) {
              if ( ! deviceIsMineBonded ) continue;
              if ( ! deviceCUDA->device_shared_with_pe(map->computeData[i].node) ) continue;
            } else {
              if ( map->computeData[i].node != myNode ) continue;
            }
          break;

          case computeBondedCUDAType:
            if ( ! deviceIsMineBonded ) continue;
            if ( map->computeData[i].node != myNode ) continue;
          break;
#endif // BONDED_CUDA

          case computeNonbondedCUDA2Type:
            if ( ! deviceIsMine ) continue;
// #ifdef BONDED_CUDA
//           case computeBondedCUDAType:
// #endif
          default:
            if ( map->computeData[i].node != myNode ) continue;
        }
#else // defined(NAMD_CUDA) || defined(NAMD_HIP)
        if ( map->computeData[i].node != myNode ) continue;
#endif
        DebugM(1,"Compute " << i << '\n');
        DebugM(1,"  node = " << map->computeData[i].node << '\n');
        DebugM(1,"  type = " << map->computeData[i].type << '\n');
        DebugM(1,"  numPids = " << map->computeData[i].numPids << '\n');
//         DebugM(1,"  numPidsAllocated = " << map->computeData[i].numPidsAllocated << '\n');
        for (int j=0; j < map->computeData[i].numPids; j++)
        {
            DebugM(1,"  pid " << map->computeData[i].pids[j].pid << '\n');
            if (!((j+1) % 6))
                DebugM(1,'\n');
        }
        DebugM(1,"\n---------------------------------------");
        DebugM(1,"---------------------------------------\n");

        createCompute(i, map);

    }

#if defined(NAMD_CUDA) || defined(NAMD_HIP)
    if (deviceIsMine) {
      getCudaComputeNonbonded()->assignPatches(this);
      getCudaComputeNonbonded()->initialize();
    }
#ifdef BONDED_CUDA
    if (simParams->bondedCUDA) {
      if (deviceIsMineBonded) {
        getComputeBondedCUDA()->initialize();
      }
    }
#endif
#endif
}

doneUpdateLocalComputes()

void ComputeMgr::doneUpdateLocalComputes

(

)

Definition at line 348 of file ComputeMgr.C.

References DebugM.

{

//  if (!--updateComputesCount) {
    DebugM(4, "doneUpdateLocalComputes on Pe("<<CkMyPe()<<")\n");
    void *msg = CkAllocMsg(0,0,0);
    CkSendMsgBranch(updateComputesReturnEP,msg,0,updateComputesReturnChareID);
//  }
}

enableComputeGlobalResults()

void ComputeMgr::enableComputeGlobalResults

(

)

Definition at line 1407 of file ComputeMgr.C.

References computeGlobalResultsMsgs, computeGlobalResultsMsgSeq, DebugM, ResizeArray< Elem >::del(), NAMD_EVENT_START, NAMD_EVENT_STOP, recvComputeGlobalResults(), and ResizeArray< Elem >::size().

Referenced by ComputeGlobal::doWork().

{
  NAMD_EVENT_START(1, NamdProfileEvent::GM_ENABLE_COMP_RESULTS);          
  ++computeGlobalResultsMsgSeq;
  DebugM(3,"["<<CkMyPe() <<"] enableComputeGlobalResults for "<< computeGlobalResultsMsgs.size() <<" messages seq "<< computeGlobalResultsMsgSeq <<"\n");
  for ( int i=0; i<computeGlobalResultsMsgs.size(); ++i ) {
    if ( computeGlobalResultsMsgs[i]->seq == computeGlobalResultsMsgSeq ) {
      ComputeGlobalResultsMsg *msg = computeGlobalResultsMsgs[i];
      computeGlobalResultsMsgs.del(i);
      recvComputeGlobalResults(msg);
      break;
    }
  }
  NAMD_EVENT_STOP(1, NamdProfileEvent::GM_ENABLE_COMP_RESULTS);
  DebugM(3,"["<<CkMyPe() <<"] exiting enableComputeGlobalResults for "<< computeGlobalResultsMsgs.size() <<" messages seq "<< computeGlobalResultsMsgSeq <<"\n");
}

getCudaGlobalMasterUpdateResult()

int ComputeMgr::getCudaGlobalMasterUpdateResult

(

const std::string &

client_name

)

const

Definition at line 1760 of file ComputeMgr.C.

                                                                                  {
  return CudaGlobalMasterClientUpdateResults.at(client_name);
}

getCudaGlobalMasterUpdateResultString()

std::string ComputeMgr::getCudaGlobalMasterUpdateResultString

(

const std::string &

client_name

)

const

Definition at line 1764 of file ComputeMgr.C.

                                                                                              {
  return CudaGlobalMasterClientUpdateResultStrings.at(client_name);
}

recvAssignPatchesOnPe()

void ComputeMgr::recvAssignPatchesOnPe

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1793 of file ComputeMgr.C.

References CudaComputeNonbonded::assignPatchesOnPe(), and CudaComputeNonbondedMsg::c.

                                                                   {
  msg->c->assignPatchesOnPe();
  delete msg;
}

recvComputeConsForceMsg()

void ComputeMgr::recvComputeConsForceMsg

(

ComputeConsForceMsg *

msg

)

Definition at line 1544 of file ComputeMgr.C.

References ComputeConsForceMsg::aid, Molecule::consForce, Molecule::consForceIndexes, ComputeConsForceMsg::f, for(), Node::molecule, Molecule::numAtoms, Node::Object(), and ResizeArray< Elem >::size().

{
    Molecule *m = Node::Object()->molecule;
    if(CkMyRank()==0){ // there is only one molecule per process
      delete [] m->consForceIndexes;
      delete [] m->consForce;
      int n = msg->aid.size();
      if (n > 0)
        {
          m->consForceIndexes = new int32[m->numAtoms];
          m->consForce = new Vector[n];
          int i;
          for (i=0; i<m->numAtoms; i++) m->consForceIndexes[i] = -1;
          for (i=0; i<msg->aid.size(); i++)
            {
              m->consForceIndexes[msg->aid[i]] = i;
              m->consForce[i] = msg->f[i];
            }
        }
      else
        {
          m->consForceIndexes = NULL;
          m->consForce = NULL;
        }
    }
    delete msg;
#ifdef NODEGROUP_FORCE_REGISTER
    if(CkMyPe()==0)
      {
        CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
        cpdata.setDeviceKernelUpdateCounter();
      }
#endif
}

recvComputeDPMEData()

void ComputeMgr::recvComputeDPMEData

(

ComputeDPMEDataMsg *

msg

)

Definition at line 1507 of file ComputeMgr.C.

References NAMD_die(), and PatchMap::Object().

{
    if ( computeDPMEObject )
    {
#ifdef DPME
        computeDPMEObject->recvData(msg);
#endif
    }
    else if ( ! (PatchMap::Object())->numHomePatches() ) delete msg;
    else NAMD_die("ComputeMgr::computeDPMEObject is NULL!");
}

recvComputeDPMEResults()

void ComputeMgr::recvComputeDPMEResults

(

ComputeDPMEResultsMsg *

msg

)

Definition at line 1525 of file ComputeMgr.C.

References NAMD_die(), and PatchMap::Object().

{
    if ( computeDPMEObject )
    {
#ifdef DPME
        computeDPMEObject->recvResults(msg);
#endif
    }
    else if ( ! (PatchMap::Object())->numHomePatches() ) delete msg;
    else NAMD_die("ComputeMgr::computeDPMEObject is NULL!");
}

recvComputeEwaldData()

void ComputeMgr::recvComputeEwaldData

(

ComputeEwaldMsg *

msg

)

Definition at line 1470 of file ComputeMgr.C.

References NAMD_die(), and ComputeEwald::recvData().

{
    if (computeEwaldObject)
        computeEwaldObject->recvData(msg);
    else NAMD_die("ComputeMgr::computeEwaldObject in recvData is NULL!");
}

recvComputeEwaldResults()

void ComputeMgr::recvComputeEwaldResults

(

ComputeEwaldMsg *

msg

)

Definition at line 1482 of file ComputeMgr.C.

References NAMD_die(), PatchMap::Object(), and ComputeEwald::recvResults().

Referenced by sendComputeEwaldResults().

{
    if (computeEwaldObject) {
        CmiEnableUrgentSend(1);
        computeEwaldObject->recvResults(msg);
        CmiEnableUrgentSend(0);
    }
    else if ( ! (PatchMap::Object())->numHomePatches() ) delete msg;
    else NAMD_die("ComputeMgr::computeEwaldObject in recvResults is NULL!");
}

recvComputeGlobalConfig()

void ComputeMgr::recvComputeGlobalConfig

(

ComputeGlobalConfigMsg *

)

recvComputeGlobalData()

void ComputeMgr::recvComputeGlobalData

(

ComputeGlobalDataMsg *

msg

)

Definition at line 1357 of file ComputeMgr.C.

References DebugM, NAMD_die(), NAMD_EVENT_START, NAMD_EVENT_STOP, and GlobalMasterServer::recvData().

{
  NAMD_EVENT_START(1, NamdProfileEvent::GM_RECV_COMP_DATA);        
    if (masterServerObject)  // make sure it has been initialized
    {
      DebugM(3, "["<<CkMyPe()<<"] recvComputeGlobalData calling recvData\n");
        masterServerObject->recvData(msg);
    }
    else NAMD_die("ComputeMgr::masterServerObject is NULL!");
    NAMD_EVENT_STOP(1, NamdProfileEvent::GM_RECV_COMP_DATA);        
}

recvComputeGlobalResults()

void ComputeMgr::recvComputeGlobalResults

(

ComputeGlobalResultsMsg *

msg

)

Definition at line 1424 of file ComputeMgr.C.

References ResizeArray< Elem >::add(), computeGlobalObject, computeGlobalResultsMsgs, computeGlobalResultsMsgSeq, DebugM, NAMD_die(), NAMD_EVENT_START, NAMD_EVENT_STOP, PatchMap::Object(), ComputeGlobal::recvResults(), and ComputeGlobalResultsMsg::seq.

Referenced by ComputeGlobal::doWork(), enableComputeGlobalResults(), and sendComputeGlobalData().

{
  NAMD_EVENT_START(1, NamdProfileEvent::GM_RCV_COMP_RESULTS);
  DebugM(3,"[" << CkMyPe() << "] recvComputeGlobalResults msg->seq "<< msg->seq << " computeGlobalResultsMsgSeq " << computeGlobalResultsMsgSeq << "\n");
    if ( computeGlobalObject )
    {
      if ( msg->seq == computeGlobalResultsMsgSeq ) {
        CmiEnableUrgentSend(1);

        computeGlobalObject->recvResults(msg);
        //      CkPrintf("*** past recvResults on PE %d \n", CkMyPe());
        CmiEnableUrgentSend(0);
      } else {
        //      CkPrintf("*** Adding recvComputeGlobalResults on PE %d \n", CkMyPe());
        computeGlobalResultsMsgs.add(msg);
      }
    }
    else if ( ! (PatchMap::Object())->numHomePatches() )
      {
        //      CkPrintf("*** ignoring recvComputeGlobalResults on PE %d due to no home patch\n", CkMyPe());
        delete msg;
      }
    else NAMD_die("ComputeMgr::computeGlobalObject is NULL!");
    NAMD_EVENT_STOP(1, NamdProfileEvent::GM_RCV_COMP_RESULTS);
    //    CkPrintf("*** exiting recvComputeGlobalResults on PE %d \n", CkMyPe());
}

recvCudaGlobalMasterCreateMsg()

void ComputeMgr::recvCudaGlobalMasterCreateMsg

(

std::vector< std::string >

args

)

Definition at line 1579 of file ComputeMgr.C.

References ComputeCUDAMgr::createCudaGlobalMaster(), DebugM, deviceCUDA, endi(), ComputeCUDAMgr::getComputeCUDAMgr(), ComputeCUDAMgr::getCudaGlobalMaster(), DeviceCUDA::getGlobalDevice(), DeviceCUDA::getIsGlobalDevice(), DeviceCUDA::getMasterPe(), iERROR(), iINFO(), iout, NAMD_die(), Node::Object(), Node::simParameters, and simParams.

                                                                        {
#if (defined(NAMD_CUDA) || defined(NAMD_HIP))
  Node *node = Node::Object();
  SimParameters *simParams = node->simParameters;
  if (simParams->CUDASOAintegrate && simParams->useCudaGlobal) {
#ifdef NODEGROUP_FORCE_REGISTER
    if (deviceCUDA->getMasterPe() == CkMyPe()) {
      if (deviceCUDA->getIsGlobalDevice()) {
        DebugM(3, "Call recvCudaGlobalMasterCreateMsg on master PE " << CkMyPe() << ".\n");
        ComputeCUDAMgr* cudaMgr = ComputeCUDAMgr::getComputeCUDAMgr();
        cudaMgr->createCudaGlobalMaster();
        std::shared_ptr<CudaGlobalMaster::CudaGlobalMasterClient> client = nullptr;
        const std::string library_name = args[0];
        // Find to see if library_name has been loaded
        std::shared_ptr<dlloader::DLLoader<CudaGlobalMaster::CudaGlobalMasterClient>> loader = nullptr;
        for (auto it = CudaGlobalMasterClientDlloaders.begin();
             it != CudaGlobalMasterClientDlloaders.end(); ++it) {
          if ((*it)->LibName() == library_name) {
            loader = (*it);
            break;
          }
        }
        // Create a new loader if not found
        if (loader == nullptr) {
          loader = std::shared_ptr<dlloader::DLLoader<CudaGlobalMaster::CudaGlobalMasterClient>>(new dlloader::DLLoader<CudaGlobalMaster::CudaGlobalMasterClient>(library_name));
        }
        try {
          iout << iINFO << "Loading library " << library_name
               << " on PE: " << CkMyPe() << "\n" << endi;
          loader->DLOpenLib();
          client = loader->DLGetInstance();
        } catch (std::exception& e) {
          iout << iERROR << "Cannot load the shared library " << library_name << "\n" << endi;
          NAMD_die(e.what());
        }
        // Try to initialize the client
        try {
          client->initialize(args,
            deviceCUDA->getGlobalDevice(),
            cudaMgr->getCudaGlobalMaster()->getStream());
          client->subscribe(cudaMgr->getCudaGlobalMaster());
          iout << iINFO << "CudaGlobalMaster client \"" << client->name()
                << "\"" << " initialized\n" << endi;
        } catch (std::exception& e) {
          iout << iERROR << "Cannot initialize the CudaGlobalMaster client from "
                << library_name << "\n" << endi;
          NAMD_die(e.what());
        }
        CudaGlobalMasterClientDlloaders.push_back(loader);
      } else {
        DebugM(3, "Skip recvCudaGlobalMasterCreateMsg on master PE " <<
                  CkMyPe() << " that is not scheduled for GPU-resident global master.\n");
      }
    } else {
      DebugM(3, "Skip recvCudaGlobalMasterCreateMsg on non-master PE " << CkMyPe() << ".\n");
    }
#endif // NODEGROUP_FORCE_REGISTER
  } else {
    if (!(simParams->CUDASOAintegrate)) {
      NAMD_die("GPU-resident mode is not enabled.\n");
    }
    if (!(simParams->useCudaGlobal)) {
      NAMD_die("GPU-resident external forces are not enabled.\n");
    }
  }
  // CmiNodeBarrier();
#endif // (defined(NAMD_CUDA) || defined(NAMD_HIP))
}

recvCudaGlobalMasterRemoveMsg()

void ComputeMgr::recvCudaGlobalMasterRemoveMsg

(

std::vector< std::string >

args

)

Definition at line 1648 of file ComputeMgr.C.

References deviceCUDA, endi(), ComputeCUDAMgr::getComputeCUDAMgr(), ComputeCUDAMgr::getCudaGlobalMaster(), DeviceCUDA::getIsGlobalDevice(), DeviceCUDA::getMasterPe(), iINFO(), iout, NAMD_die(), Node::Object(), Node::simParameters, and simParams.

                                                                        {
#if (defined(NAMD_CUDA) || defined(NAMD_HIP))
  Node *node = Node::Object();
  SimParameters *simParams = node->simParameters;
  const std::string client_name_to_remove = args[0];
  if (simParams->CUDASOAintegrate && simParams->useCudaGlobal) {
#ifdef NODEGROUP_FORCE_REGISTER
    if (deviceCUDA->getMasterPe() == CkMyPe()) {
      if (deviceCUDA->getIsGlobalDevice()) {
        ComputeCUDAMgr* cudaMgr = ComputeCUDAMgr::getComputeCUDAMgr();
        std::shared_ptr<CudaGlobalMasterServer> gm = cudaMgr->getCudaGlobalMaster();
        if (gm) {
          std::shared_ptr<CudaGlobalMaster::CudaGlobalMasterClient> c = nullptr;
          const std::vector<std::shared_ptr<CudaGlobalMaster::CudaGlobalMasterClient>>& clients = gm->getClients();
          for (size_t i = 0; i < clients.size(); ++i) {
            if (client_name_to_remove == clients[i]->name()) {
              c = clients[i];
              break;
            }
          }
          if (c) {
            gm->removeClient(c);
            iout << iINFO << "CudaGlobalMasterClient \""
                 << client_name_to_remove << "\" removed\n" << endi;
          } else {
            const std::string error = "CudaGlobalMasterClient \""
              + client_name_to_remove + "\" not found";
            NAMD_die(error.c_str());
          }
        }
      }
    }
#endif // NODEGROUP_FORCE_REGISTER
  } else {
    if (!(simParams->CUDASOAintegrate)) {
      NAMD_die("GPU-resident mode is not enabled.\n");
    }
    if (!(simParams->useCudaGlobal)) {
      NAMD_die("GPU-resident external forces are not enabled.\n");
    }
  }
#endif // (defined(NAMD_CUDA) || defined(NAMD_HIP))
}

recvCudaGlobalMasterUpdateMsg()

void ComputeMgr::recvCudaGlobalMasterUpdateMsg

(

std::vector< std::string >

args

)

Definition at line 1692 of file ComputeMgr.C.

References deviceCUDA, endi(), ComputeCUDAMgr::getComputeCUDAMgr(), ComputeCUDAMgr::getCudaGlobalMaster(), DeviceCUDA::getIsGlobalDevice(), DeviceCUDA::getMasterPe(), iINFO(), iout, NAMD_die(), Node::Object(), Node::simParameters, and simParams.

                                                                        {
#if (defined(NAMD_CUDA) || defined(NAMD_HIP))
  std::vector<std::string> result_args;
  Node *node = Node::Object();
  SimParameters *simParams = node->simParameters;
  const std::string client_name_to_update = args[0];
#ifdef NAMD_TCL
  int error_code = TCL_OK;
#else
  int error_code = 0;
#endif
  if (simParams->CUDASOAintegrate && simParams->useCudaGlobal) {
#ifdef NODEGROUP_FORCE_REGISTER
    if (deviceCUDA->getMasterPe() == CkMyPe()) {
      if (deviceCUDA->getIsGlobalDevice()) {
        ComputeCUDAMgr* cudaMgr = ComputeCUDAMgr::getComputeCUDAMgr();
        std::shared_ptr<CudaGlobalMasterServer> gm = cudaMgr->getCudaGlobalMaster();
        if (gm) {
          std::shared_ptr<CudaGlobalMaster::CudaGlobalMasterClient> c = nullptr;
          const std::vector<std::shared_ptr<CudaGlobalMaster::CudaGlobalMasterClient>>& clients = gm->getClients();
          for (size_t i = 0; i < clients.size(); ++i) {
            if (client_name_to_update == clients[i]->name()) {
              c = clients[i];
              break;
            }
          }
          if (c) {
            result_args.push_back(client_name_to_update);
            error_code = c->updateFromTCLCommand(args);
            result_args.push_back(c->getTCLUpdateResult());
            iout << iINFO << "CudaGlobalMasterClient \""
                 << client_name_to_update << "\" updated\n" << endi;
          } else {
            const std::string error = "CudaGlobalMasterClient \""
              + client_name_to_update + "\" not found";
            NAMD_die(error.c_str());
          }
        }
      }
    }
#endif // NODEGROUP_FORCE_REGISTER
  } else {
    if (!(simParams->CUDASOAintegrate)) {
      NAMD_die("GPU-resident mode is not enabled.\n");
    }
    if (!(simParams->useCudaGlobal)) {
      NAMD_die("GPU-resident external forces are not enabled.\n");
    }
  }
  CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
  cm[0].recvCudaGlobalMasterUpdateResultMsg(error_code, result_args);
#endif // (defined(NAMD_CUDA) || defined(NAMD_HIP))
}

recvCudaGlobalMasterUpdateResultMsg()

void ComputeMgr::recvCudaGlobalMasterUpdateResultMsg	(	int	tcl_error_code,
		std::vector< std::string >	args
	)

Definition at line 1746 of file ComputeMgr.C.

References NAMD_bug().

                                                                                                  {
  if (CkMyPe() == 0) {
    if (!args.empty()) {
      CudaGlobalMasterClientUpdateResults[args[0]] = tcl_error_code;
      CudaGlobalMasterClientUpdateResultStrings[args[0]] = args[1];
    }
  } else {
    const std::string error =
      "recvCudaGlobalMasterUpdateResultMsg is called on " +
      std::to_string(CkMyPe()) + " but expected on PE 0!\n";
    NAMD_bug(error.c_str());
  }
}

recvFinishPatchesOnPe()

void ComputeMgr::recvFinishPatchesOnPe

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1820 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, and CudaComputeNonbonded::finishPatchesOnPe().

                                                                   {
  msg->c->finishPatchesOnPe();
  delete msg;
}

recvFinishPatchOnPe()

void ComputeMgr::recvFinishPatchOnPe

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1833 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, CudaComputeNonbonded::finishPatchOnPe(), and CudaComputeNonbondedMsg::i.

                                                                 {
  msg->c->finishPatchOnPe(msg->i);
  delete msg;
}

recvFinishReductions()

void ComputeMgr::recvFinishReductions

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1858 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, and CudaComputeNonbonded::finishReductions().

                                                                  {
  msg->c->finishReductions();
  delete msg;
}

recvLaunchWork()

void ComputeMgr::recvLaunchWork

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1880 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, and CudaComputeNonbonded::launchWork().

                                                            {
  msg->c->launchWork();
  delete msg;
}

recvMessageEnqueueWork()

void ComputeMgr::recvMessageEnqueueWork

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1869 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, and CudaComputeNonbonded::messageEnqueueWork().

                                                                    {
  msg->c->messageEnqueueWork();
  delete msg;
}

recvOpenBoxesOnPe()

void ComputeMgr::recvOpenBoxesOnPe

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1847 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, and CudaComputeNonbonded::openBoxesOnPe().

                                                               {
  msg->c->openBoxesOnPe();
  delete msg;
}

recvSkipPatchesOnPe()

void ComputeMgr::recvSkipPatchesOnPe

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1806 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, and CudaComputeNonbonded::skipPatchesOnPe().

                                                                 {
  msg->c->skipPatchesOnPe();
  delete msg;
}

recvUnregisterBoxesOnPe()

void ComputeMgr::recvUnregisterBoxesOnPe

(

CudaComputeNonbondedMsg *

msg

)

Definition at line 1893 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, and CudaComputeNonbonded::unregisterBoxesOnPe().

                                                                     {
  msg->c->unregisterBoxesOnPe();
  delete msg;
}

recvYieldDevice()

void ComputeMgr::recvYieldDevice

(

int

pe

)

Definition at line 1773 of file ComputeMgr.C.

                                       {
  // XXX MIC support was only code using YieldDevice functionality
  // computeNonbondedMICObject->recvYieldDevice(pe);
}

sendAssignPatchesOnPe()

void ComputeMgr::sendAssignPatchesOnPe	(	std::vector< int > &	pes,
		CudaComputeNonbonded *	c
	)

Definition at line 1785 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c.

Referenced by CudaComputeNonbonded::assignPatches().

                                                                                   {
  for (int i=0;i < pes.size();i++) {
    CudaComputeNonbondedMsg *msg = new CudaComputeNonbondedMsg;
    msg->c = c;
    thisProxy[pes[i]].recvAssignPatchesOnPe(msg);
  }
}

sendComputeDPMEData()

void ComputeMgr::sendComputeDPMEData

(

ComputeDPMEDataMsg *

msg

)

Definition at line 1493 of file ComputeMgr.C.

References NAMD_die(), and PatchMap::Object().

{
    if ( computeDPMEObject )
    {
#ifdef DPME
        int node = computeDPMEObject->getMasterNode();
        CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
        cm.recvComputeDPMEData(msg,node);
#endif
    }
    else if ( ! (PatchMap::Object())->numHomePatches() ) delete msg;
    else NAMD_die("ComputeMgr::computeDPMEObject is NULL!");
}

sendComputeDPMEResults()

void ComputeMgr::sendComputeDPMEResults	(	ComputeDPMEResultsMsg *	msg,
		int	node
	)

Definition at line 1519 of file ComputeMgr.C.

{
    CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
    cm[node].recvComputeDPMEResults(msg);
}

sendComputeEwaldData()

void ComputeMgr::sendComputeEwaldData

(

ComputeEwaldMsg *

msg

)

Definition at line 1454 of file ComputeMgr.C.

References ComputeEwald::getMasterNode(), NAMD_die(), and PatchMap::Object().

Referenced by ComputeEwald::doWork().

{
    if (computeEwaldObject)
    {
        int node = computeEwaldObject->getMasterNode();
        CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
        cm[node].recvComputeEwaldData(msg);
    }
    else if (!PatchMap::Object()->numHomePatches())
    {
      // CkPrintf("skipping message on Pe(%d)\n", CkMyPe());
        delete msg;
    }
    else NAMD_die("ComputeMgr::computeEwaldObject is NULL!");
}

sendComputeEwaldResults()

void ComputeMgr::sendComputeEwaldResults

(

ComputeEwaldMsg *

msg

)

Definition at line 1477 of file ComputeMgr.C.

References recvComputeEwaldResults().

Referenced by ComputeEwald::recvData().

{
    (CProxy_ComputeMgr(CkpvAccess(BOCclass_group).computeMgr)).recvComputeEwaldResults(msg);
}

sendComputeGlobalConfig()

void ComputeMgr::sendComputeGlobalConfig

(

ComputeGlobalConfigMsg *

)

sendComputeGlobalData()

void ComputeMgr::sendComputeGlobalData

(

ComputeGlobalDataMsg *

msg

)

Definition at line 1294 of file ComputeMgr.C.

References SimParameters::CUDASOAintegrate, DebugM, NAMD_EVENT_START, NAMD_EVENT_STOP, Node::Object(), recvComputeGlobalResults(), and Node::simParameters.

Referenced by ComputeGlobal::doWork().

{
  NAMD_EVENT_START(1, NamdProfileEvent::GM_SEND_COMP_DATA);        
  //  CkPrintf("*** [%d] Calling sendComputeGlobalData\n", CkMyPe());
  #ifdef NODEGROUP_FORCE_REGISTER
  SimParameters *sp = Node::Object()->simParameters;
  if (sp->CUDASOAintegrate) {
    NAMD_EVENT_START(1, NamdProfileEvent::GM_NODELOCK);        
    CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
    PatchData *patchData = cpdata.ckLocalBranch();
    CmiNodeLock &nl = patchData->nodeLock;
    // atomic access to GlobalMasterServer to simulate queueing
    if (CkMyPe() != 0)
    {
      CmiLock(nl);
      //CkPrintf("*** [%d] Acquired nodelock!\n", CkMyPe());
      patchData->master_mgr->recvComputeGlobalData(msg);
      CmiUnlock(nl);
    }
    NAMD_EVENT_STOP(1, NamdProfileEvent::GM_NODELOCK);
    NAMD_EVENT_START(1, NamdProfileEvent::GM_BARRIER);        
    // Barrier to make sure 0 goes last, since invocation of the clients and
    // message coordination has to happen on PE 0 and the last PE to call
    // recvComputeGlobalData will trigger all of that on itself
    //    CmiNodeBarrier();
    //    CkPrintf("*** sendComputeGlobalData entering barrier 1 on PE %d \n", CkMyPe());
    stowSuspendULT();
    
    NAMD_EVENT_STOP(1, NamdProfileEvent::GM_BARRIER);        
    if (CkMyPe() == 0)
      {
        CmiLock(nl);
        patchData->master_mgr->recvComputeGlobalData(msg);
        CmiUnlock(nl);
      }
    else
    {
      // All PEs other than 0 wait here while the clients run and the global
      // results messages are prepared and copied into their slots (happens from
      // sendComputeGlobalResults on PE0)
      //  CmiNodeBarrier();
      //      CkPrintf("before call to stow %d\n",CkMyPe());
      //      CkPrintf("*** sendComputeGlobalData barrier 3 on PE %d \n", CkMyPe());
      stowSuspendULT();
      //      CkPrintf("*** sendComputeGlobalData out barrier 3 on PE %d \n", CkMyPe());
      //      CkPrintf("returned from call to stow %d\n",CkMyPe());
    }
    // Get the message from the slot for this PE and resume execution
    ComputeGlobalResultsMsg* resultsMsg = CkpvAccess(ComputeGlobalResultsMsg_instance);
    DebugM(3,"["<<CkMyPe()<<"] calling recvComputeGlobalResults\n");
    recvComputeGlobalResults(resultsMsg);
  } else {
  #endif
    CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
    DebugM(3,"["<<CkMyPe()<<"] msg to recvComputeGlobalData\n");
    cm[0].recvComputeGlobalData(msg);
  #ifdef NODEGROUP_FORCE_REGISTER
  }
  #endif
  NAMD_EVENT_STOP(1, NamdProfileEvent::GM_SEND_COMP_DATA);
  DebugM(3,"["<<CkMyPe()<<"] done sendComputeGlobalData\n");
}

sendComputeGlobalResults()

void ComputeMgr::sendComputeGlobalResults

(

ComputeGlobalResultsMsg *

msg

)

Definition at line 1369 of file ComputeMgr.C.

References computeGlobalResultsMsgMasterSeq, SimParameters::CUDASOAintegrate, DebugM, NAMD_EVENT_START, NAMD_EVENT_STOP, Node::Object(), ComputeGlobalResultsMsg::seq, and Node::simParameters.

{
  NAMD_EVENT_START(1, NamdProfileEvent::GM_SEND_COMP_RESULTS);        
    msg->seq = ++computeGlobalResultsMsgMasterSeq;
    DebugM(3,"["<< CkMyPe()<< "] sendComputeGlobalResults seq "<<msg->seq<<"\n");

  #ifdef NODEGROUP_FORCE_REGISTER
  SimParameters *sp = Node::Object()->simParameters;
  if (sp->CUDASOAintegrate) {
    // Only PE 0 runs this code
    // Copy the message into each PE's slot (Assumes single-node with multicore build)
    for (int pe = 0; pe < CkMyNodeSize(); pe++) {
      if(CkpvAccessOther(ComputeGlobalResultsMsg_instance, pe)!=nullptr)
        {
          // make sure msg delete happens on the same PE as made the msg to
          // avoid unbounded memory pool growth for these unsent messages
          delete CkpvAccessOther(ComputeGlobalResultsMsg_instance, pe);
        }
      CkpvAccessOther(ComputeGlobalResultsMsg_instance, pe) = (ComputeGlobalResultsMsg*)CkCopyMsg((void**)&msg);
    }
    delete msg;
    // Now that copies are done, trigger the barrier to resume the other PEs
    // (most other PEs call this barrier from sendComputeGlobalData)
    //    CkPrintf("this is where we would call awaken\n",CkMyPe());
    //CmiNodeBarrier();
    //    CkPrintf("*** sendComputeGlobalResults entering barrier 2 on PE %d \n", CkMyPe());
    stowSuspendULT();
    //thisProxy.recvComputeGlobalResults(msg);
  } else {
  #endif
    DebugM(3,"["<< CkMyPe() << "] ComputeMgr::sendComputeGlobalResults invoking bcast recvComputeGlobalResults\n");    
    thisProxy.recvComputeGlobalResults(msg);
  #ifdef NODEGROUP_FORCE_REGISTER
  }
  #endif
  NAMD_EVENT_STOP(1, NamdProfileEvent::GM_SEND_COMP_RESULTS);        
}

sendFinishPatchesOnPe()

void ComputeMgr::sendFinishPatchesOnPe	(	std::vector< int > &	pes,
		CudaComputeNonbonded *	c
	)

Definition at line 1811 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, COMPUTE_PROXY_PRIORITY, PRIORITY_SIZE, Compute::sequence(), and SET_PRIORITY.

Referenced by CudaComputeNonbonded::finishPatches().

                                                                                   {
  for (int i=0;i < pes.size();i++) {
    CudaComputeNonbondedMsg *msg = new (PRIORITY_SIZE) CudaComputeNonbondedMsg;
    SET_PRIORITY(msg, c->sequence(), COMPUTE_PROXY_PRIORITY);
    msg->c = c;
    thisProxy[pes[i]].recvFinishPatchesOnPe(msg);
  }
}

sendFinishPatchOnPe()

void ComputeMgr::sendFinishPatchOnPe	(	int	pe,
		CudaComputeNonbonded *	c,
		int	i,
		PatchID	patchID
	)

Definition at line 1825 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, COMPUTE_PROXY_PRIORITY, CudaComputeNonbondedMsg::i, PATCH_PRIORITY, PRIORITY_SIZE, Compute::sequence(), and SET_PRIORITY.

                                                                                            {
  CudaComputeNonbondedMsg *msg = new (PRIORITY_SIZE) CudaComputeNonbondedMsg;
  SET_PRIORITY(msg, c->sequence(), COMPUTE_PROXY_PRIORITY + PATCH_PRIORITY(patchID));
  msg->c = c;
  msg->i = i;
  thisProxy[pe].recvFinishPatchOnPe(msg);
}

sendFinishReductions()

void ComputeMgr::sendFinishReductions	(	int	pe,
		CudaComputeNonbonded *	c
	)

Definition at line 1852 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c.

Referenced by CudaComputeNonbonded::skipPatchesOnPe().

                                                                     {
  CudaComputeNonbondedMsg *msg = new CudaComputeNonbondedMsg;
  msg->c = c;
  thisProxy[pe].recvFinishReductions(msg);
}

sendLaunchWork()

void ComputeMgr::sendLaunchWork	(	int	pe,
		CudaComputeNonbonded *	c
	)

Definition at line 1874 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c.

Referenced by CudaComputeNonbonded::openBoxesOnPe().

                                                               {
  CudaComputeNonbondedMsg *msg = new CudaComputeNonbondedMsg;
  msg->c = c;
  thisProxy[pe].recvLaunchWork(msg);
}

sendMessageEnqueueWork()

void ComputeMgr::sendMessageEnqueueWork	(	int	pe,
		CudaComputeNonbonded *	c
	)

Definition at line 1863 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c.

Referenced by CudaComputeNonbonded::noWork().

                                                                       {
  CudaComputeNonbondedMsg *msg = new CudaComputeNonbondedMsg;
  msg->c = c;
  thisProxy[pe].recvMessageEnqueueWork(msg);
}

sendOpenBoxesOnPe()

void ComputeMgr::sendOpenBoxesOnPe	(	std::vector< int > &	pes,
		CudaComputeNonbonded *	c
	)

Definition at line 1838 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c, PRIORITY_SIZE, PROXY_DATA_PRIORITY, Compute::sequence(), and SET_PRIORITY.

Referenced by CudaComputeNonbonded::doWork().

                                                                               {
  for (int i=0;i < pes.size();i++) {
    CudaComputeNonbondedMsg *msg = new (PRIORITY_SIZE) CudaComputeNonbondedMsg;
    SET_PRIORITY(msg, c->sequence(), PROXY_DATA_PRIORITY+1); // after bonded
    msg->c = c;
    thisProxy[pes[i]].recvOpenBoxesOnPe(msg);
  }
}

sendSkipPatchesOnPe()

void ComputeMgr::sendSkipPatchesOnPe	(	std::vector< int > &	pes,
		CudaComputeNonbonded *	c
	)

Definition at line 1798 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c.

                                                                                 {
  for (int i=0;i < pes.size();i++) {
    CudaComputeNonbondedMsg *msg = new CudaComputeNonbondedMsg;
    msg->c = c;
    thisProxy[pes[i]].recvSkipPatchesOnPe(msg);
  }
}

sendUnregisterBoxesOnPe()

void ComputeMgr::sendUnregisterBoxesOnPe	(	std::vector< int > &	pes,
		CudaComputeNonbonded *	c
	)

Definition at line 1885 of file ComputeMgr.C.

References CudaComputeNonbondedMsg::c.

Referenced by CudaComputeNonbonded::~CudaComputeNonbonded().

                                                                                     {
  for (int i=0;i < pes.size();i++) {
    CudaComputeNonbondedMsg *msg = new CudaComputeNonbondedMsg;
    msg->c = c;
    thisProxy[pes[i]].recvUnregisterBoxesOnPe(msg);
  }
}

sendYieldDevice()

void ComputeMgr::sendYieldDevice

(

int

pe

)

Definition at line 1768 of file ComputeMgr.C.

                                       {
    CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
    cm[pe].recvYieldDevice(CkMyPe());
}

splitComputes()

void ComputeMgr::splitComputes

(

)

Definition at line 175 of file ComputeMgr.C.

References ComputeMap::cloneCompute(), ComputeMap::extendPtrs(), ComputeMap::newNode(), ComputeMap::newNumPartitions(), ComputeMap::node(), ComputeMap::numComputes(), ComputeMap::numPartitions(), ComputeMap::Object(), ComputeMap::setNewNode(), ComputeMap::setNewNumPartitions(), and ComputeMap::setNumPartitions().

{
  if ( ! CkMyRank() ) {
    ComputeMap *computeMap = ComputeMap::Object();
    const int nc = computeMap->numComputes();

    for (int i=0; i<nc; i++) {
      int nnp = computeMap->newNumPartitions(i);
      if ( nnp > 0 ) {
        if ( computeMap->numPartitions(i) != 1 ) {
          CkPrintf("Warning: unable to partition compute %d\n", i);
          computeMap->setNewNumPartitions(i,0);
          continue;
        }
        //CkPrintf("splitting compute %d by %d\n",i,nnp);
        computeMap->setNumPartitions(i,nnp);
        if (computeMap->newNode(i) == -1) {
          computeMap->setNewNode(i,computeMap->node(i));
        }
        for ( int j=1; j<nnp; ++j ) {
          int newcid = computeMap->cloneCompute(i,j);
          //CkPrintf("compute %d partition %d is %d\n",i,j,newcid);
        }
      }
    }
    computeMap->extendPtrs();
  }

  if (!CkMyPe())
  {
    CkStartQD(CkIndex_ComputeMgr::splitComputes2((CkQdMsg*)0), &thishandle);
  }
}

splitComputes2()

void ComputeMgr::splitComputes2

(

CkQdMsg *

msg

)

Definition at line 209 of file ComputeMgr.C.

{
    delete msg;
    CProxy_ComputeMgr(thisgroup).updateLocalComputes();
}

updateComputes()

void ComputeMgr::updateComputes	(	int	ep,
		CkGroupID	chareID
	)

Definition at line 142 of file ComputeMgr.C.

References NAMD_bug().

Referenced by LdbCoordinator::ExecuteMigrations().

{
    updateComputesReturnEP = ep;
    updateComputesReturnChareID = chareID;
    updateComputesCount = CkNumPes();

    if (CkMyPe())
    {
        NAMD_bug("updateComputes signaled on wrong Pe!");
    }

    CkStartQD(CkIndex_ComputeMgr::updateComputes2((CkQdMsg*)0),&thishandle);
}

updateComputes2()

void ComputeMgr::updateComputes2

(

CkQdMsg *

msg

)

Definition at line 156 of file ComputeMgr.C.

References WorkDistrib::saveComputeMapChanges().

{
    delete msg;

    CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
    WorkDistrib  *workDistrib = wd.ckLocalBranch();
    workDistrib->saveComputeMapChanges(CkIndex_ComputeMgr::updateComputes3(),thisgroup);
}

updateComputes3()

void ComputeMgr::updateComputes3

(

)

Definition at line 165 of file ComputeMgr.C.

{
    if ( skipSplitting ) {
      CProxy_ComputeMgr(thisgroup).updateLocalComputes();
    } else {
      CProxy_ComputeMgr(thisgroup).splitComputes();
      skipSplitting = 1;
    }
}

updateLocalComputes()

void ComputeMgr::updateLocalComputes

(

)

Definition at line 215 of file ComputeMgr.C.

References ResizeArray< Elem >::add(), ComputeMap::compute(), ProxyMgr::createProxy(), Compute::ldObjHandle, LdbCoordinator::Migrate(), ComputeMap::newNode(), ComputeMap::newNumPartitions(), ComputeMap::node(), ComputeMap::numComputes(), ComputeMap::numPids(), ComputeMap::Object(), LdbCoordinator::Object(), ComputeMap::pid(), ComputeMap::registerCompute(), and ResizeArray< Elem >::resize().

{
    ComputeMap *computeMap = ComputeMap::Object();
    CProxy_ProxyMgr pm(CkpvAccess(BOCclass_group).proxyMgr);
    ProxyMgr *proxyMgr = pm.ckLocalBranch();
    LdbCoordinator *ldbCoordinator = LdbCoordinator::Object();

     computeFlag.resize(0);

    const int nc = computeMap->numComputes();
    for (int i=0; i<nc; i++) {

        if ( computeMap->node(i) == CkMyPe() &&
             computeMap->newNumPartitions(i) > 1 ) {
           Compute *c = computeMap->compute(i);
           ldbCoordinator->Migrate(c->ldObjHandle,CkMyPe());
           delete c;
           computeMap->registerCompute(i,NULL);
           if ( computeMap->newNode(i) == CkMyPe() ) computeFlag.add(i); 
        } else
        if (computeMap->newNode(i) == CkMyPe() && computeMap->node(i) != CkMyPe())
        {
            computeFlag.add(i);
            for (int n=0; n < computeMap->numPids(i); n++)
            {
                proxyMgr->createProxy(computeMap->pid(i,n));
            }
        }
        else if (computeMap->node(i) == CkMyPe() &&
                 (computeMap->newNode(i) != -1 && computeMap->newNode(i) != CkMyPe() ))
        {
            // CkPrintf("delete compute %d on pe %d\n",i,CkMyPe());
            delete computeMap->compute(i);
            computeMap->registerCompute(i,NULL);
        }
    }

    if (!CkMyPe())
    {
        CkStartQD(CkIndex_ComputeMgr::updateLocalComputes2((CkQdMsg*)0), &thishandle);
    }
}

updateLocalComputes2()

void ComputeMgr::updateLocalComputes2

(

CkQdMsg *

msg

)

Definition at line 259 of file ComputeMgr.C.

{
    delete msg;
    CProxy_ComputeMgr(thisgroup).updateLocalComputes3();
}

updateLocalComputes3()

void ComputeMgr::updateLocalComputes3

(

)

Definition at line 266 of file ComputeMgr.C.

References ResizeArray< Elem >::clear(), ComputeMap::newNode(), ProxyMgr::nodecount, ComputeMap::numComputes(), ComputeMap::Object(), ProxyMgr::removeUnusedProxies(), ComputeMap::setNewNode(), ComputeMap::setNewNumPartitions(), ComputeMap::setNode(), and ResizeArray< Elem >::size().

{
    ComputeMap *computeMap = ComputeMap::Object();
    CProxy_ProxyMgr pm(CkpvAccess(BOCclass_group).proxyMgr);
    ProxyMgr *proxyMgr = pm.ckLocalBranch();

    ProxyMgr::nodecount = 0;

    const int nc = computeMap->numComputes();

    if ( ! CkMyRank() ) {
      for (int i=0; i<nc; i++) {
        computeMap->setNewNumPartitions(i,0);
        if (computeMap->newNode(i) != -1) {
          computeMap->setNode(i,computeMap->newNode(i));
          computeMap->setNewNode(i,-1);
        }
      }
    }
 
    for(int i=0; i<computeFlag.size(); i++) createCompute(computeFlag[i], computeMap);
    computeFlag.clear();

    proxyMgr->removeUnusedProxies();

    if (!CkMyPe())
    {
        CkStartQD(CkIndex_ComputeMgr::updateLocalComputes4((CkQdMsg*)0), &thishandle);
    }
}

updateLocalComputes4()

void ComputeMgr::updateLocalComputes4

(

CkQdMsg *

msg

)

Definition at line 298 of file ComputeMgr.C.

References Node::Object(), ComputeMap::Object(), ComputeMap::saveComputeMap(), Node::simParameters, and simParams.

{
    delete msg;
    CProxy_ComputeMgr(thisgroup).updateLocalComputes5();

    // store the latest compute map
           SimParameters *simParams = Node::Object()->simParameters;
    if (simParams->storeComputeMap) {
      ComputeMap *computeMap = ComputeMap::Object();
      computeMap->saveComputeMap(simParams->computeMapFilename);
    }
}

updateLocalComputes5()

void ComputeMgr::updateLocalComputes5

(

)

Definition at line 316 of file ComputeMgr.C.

References ProxyMgr::buildProxySpanningTree2(), PatchMap::checkMap(), ComputeMap::checkMap(), PatchMap::Object(), ComputeMap::Object(), ProxyMgr::Object(), proxyRecvSpanning, proxySendSpanning, and ProxyMgr::sendSpanningTrees().

{
    if ( ! CkMyRank() ) {
      ComputeMap::Object()->checkMap();
      PatchMap::Object()->checkMap();
    }

    // we always use the centralized building of spanning tree
    // distributed building of ST called in Node.C only
    if (proxySendSpanning || proxyRecvSpanning)
        ProxyMgr::Object()->buildProxySpanningTree2();

    // this code needs to be turned on if we want to
    // shift the creation of ST to the load balancer

#if 0
    if (proxySendSpanning || proxyRecvSpanning)
    {
        if (firstphase)
            ProxyMgr::Object()->buildProxySpanningTree2();
        else
            if (CkMyPe() == 0)
                ProxyMgr::Object()->sendSpanningTrees();

        firstphase = 0;
    }
#endif

    if (!CkMyPe())
        CkStartQD(CkIndex_ComputeMgr::doneUpdateLocalComputes(), &thishandle);
}

Member Data Documentation

callMeBackCB

CkCallback ComputeMgr::callMeBackCB

Definition at line 164 of file ComputeMgr.h.

computeGlobalObject

ComputeGlobal* ComputeMgr::computeGlobalObject

Definition at line 160 of file ComputeMgr.h.

Referenced by ComputeMgr(), Sequencer::integrate(), Sequencer::integrate_SOA(), Sequencer::minimize(), and recvComputeGlobalResults().

computeGlobalResultsMsgMasterSeq

int ComputeMgr::computeGlobalResultsMsgMasterSeq

Definition at line 163 of file ComputeMgr.h.

Referenced by ComputeMgr(), and sendComputeGlobalResults().

computeGlobalResultsMsgs

ResizeArray<ComputeGlobalResultsMsg*> ComputeMgr::computeGlobalResultsMsgs

Definition at line 161 of file ComputeMgr.h.

Referenced by enableComputeGlobalResults(), and recvComputeGlobalResults().

computeGlobalResultsMsgSeq

int ComputeMgr::computeGlobalResultsMsgSeq

Definition at line 162 of file ComputeMgr.h.

Referenced by ComputeMgr(), enableComputeGlobalResults(), and recvComputeGlobalResults().

---

The documentation for this class was generated from the following files:

• ComputeMgr.h
• ComputeMgr.C