DeviceCUDA.C File Reference

#include <cuda_runtime.h>
#include <cuda.h>
#include "common.h"
#include "charm++.h"
#include "DeviceCUDA.h"
#include "WorkDistrib.h"
#include "CudaUtils.h"
#include "Node.h"
#include "SimParameters.h"

Go to the source code of this file.

Classes
struct	cuda_args_t

Macros
#define	MAX_NUM_RANKS   2048
#define	MAX_NUM_DEVICES   256
#define	REGISTER_DEVICE_EVENTS(DEV)

Functions
void	cuda_finalize ()
void	cuda_initialize ()
void	cuda_getargs (char **argv)

Variables
__thread DeviceCUDA *	deviceCUDA
static __thread cuda_args_t	cuda_args
int	deviceIDList [MAX_NUM_RANKS]
int	masterPeList [MAX_NUM_DEVICES]

Macro Definition Documentation

MAX_NUM_DEVICES

#define MAX_NUM_DEVICES   256

Definition at line 94 of file DeviceCUDA.C.

Referenced by DeviceCUDA::initialize().

MAX_NUM_RANKS

#define MAX_NUM_RANKS   2048

Definition at line 91 of file DeviceCUDA.C.

Referenced by DeviceCUDA::initialize().

REGISTER_DEVICE_EVENTS

#define REGISTER_DEVICE_EVENTS

(

DEV

)

Value:

traceRegisterUserEvent("CUDA device " #DEV " remote", CUDA_EVENT_ID_BASE + 2 * DEV); \
  traceRegisterUserEvent("CUDA device " #DEV " local", CUDA_EVENT_ID_BASE + 2 * DEV + 1);

Function Documentation

cuda_finalize()

void cuda_finalize

(

)

Definition at line 36 of file DeviceCUDA.C.

Referenced by cuda_initialize().

                     {
#if  defined(NAMD_CUDA) || ((NAMD_HIP) && ((HIP_VERSION_MAJOR < 4) && (HIP_VERSION_MINOR < 5)))
    int ndevs = 0;
    cudaGetDeviceCount(&ndevs);
    for ( int dev=0; dev < ndevs; ++dev ) {
        cudaSetDevice(dev);
        cudaDeviceReset();
    }
#else
  // for hip on rocm versions later > 4.5, there seems to be 
  // additional hipFree's called after hipDeviceReset, so removing it for now
#endif
}

cuda_getargs()

void cuda_getargs

(

char **

argv

)

Definition at line 69 of file DeviceCUDA.C.

References cuda_args, cuda_args_t::devicelist, cuda_args_t::devicesperreplica, cuda_args_t::globaldevice, cuda_args_t::ignoresharing, cuda_args_t::mergegrids, NAMD_die(), cuda_args_t::nomergegrids, cuda_args_t::nostreaming, cuda_args_t::pmedevice, cuda_args_t::pmePes, and cuda_args_t::usedevicelist.

Referenced by all_init().

                               {
  cuda_args.devicelist = 0;
  cuda_args.usedevicelist = CmiGetArgStringDesc(argv, "+devices", &cuda_args.devicelist,
                "comma-delimited list of CUDA device numbers such as 0,2,1,2");
  cuda_args.devicesperreplica = 0;
  CmiGetArgInt(argv, "+devicesperreplica", &cuda_args.devicesperreplica);
  if ( cuda_args.devicesperreplica < 0 ) NAMD_die("Devices per replica must be positive\n");
  cuda_args.ignoresharing = CmiGetArgFlag(argv, "+ignoresharing");
  cuda_args.mergegrids = CmiGetArgFlag(argv, "+mergegrids");
  cuda_args.nomergegrids = CmiGetArgFlag(argv, "+nomergegrids");
  if ( cuda_args.mergegrids && cuda_args.nomergegrids ) NAMD_die("Do not specify both +mergegrids and +nomergegrids");
  cuda_args.nostreaming = CmiGetArgFlag(argv, "+nostreaming");
#ifdef NODEGROUP_FORCE_REGISTER
  CmiGetArgInt(argv, "+pmedevice", &cuda_args.pmedevice);
  CmiGetArgInt(argv, "+pmepes", &cuda_args.pmePes);
  cuda_args.globaldevice = -1;
  CmiGetArgInt(argv, "+globaldevice", &cuda_args.globaldevice);
#endif
}

cuda_initialize()

void cuda_initialize

(

)

Definition at line 27 of file DeviceCUDA.C.

References cuda_finalize(), deviceCUDA, and DeviceCUDA::initialize().

Referenced by WorkDistrib::peOrderingReady().

                       {
    deviceCUDA = new DeviceCUDA();
    deviceCUDA->initialize();
    if (CkMyRank() == 0) {
      std::atexit(cuda_finalize);
    }
}

Variable Documentation

cuda_args

__thread cuda_args_t cuda_args

static

Definition at line 67 of file DeviceCUDA.C.

Referenced by cuda_getargs(), and DeviceCUDA::initialize().

deviceCUDA

__thread DeviceCUDA* deviceCUDA

Definition at line 23 of file DeviceCUDA.C.

Referenced by CudaComputeNonbonded::assignPatches(), CudaPmeOneDevice::checkPatchLevelSimParamCompatibility(), ComputeMgr::createComputes(), ComputeCUDAMgr::createCudaComputeNonbonded(), ComputeCUDAMgr::createCudaGlobalMaster(), ComputeCUDAMgr::createCudaPmeOneDevice(), cuda_initialize(), ComputePme::doWork(), ComputeCUDAMgr::getCudaComputeNonbonded(), GlobalGPUMgr::getIsPmeDevice(), globalAtomIDToSOAID(), ComputeCUDAMgr::initialize(), GlobalGPUMgr::initialize(), CudaComputeNonbonded::initialize(), ComputePmeMgr::initialize(), ComputePmeMgr::initialize_computes(), ComputePmeMgr::initialize_pencils(), ComputePmeCUDAMgr::initialize_pencils(), ComputePmeCUDAMgr::isPmeDevice(), ComputeMgr::recvCudaGlobalMasterCreateMsg(), ComputeMgr::recvCudaGlobalMasterRemoveMsg(), ComputeMgr::recvCudaGlobalMasterUpdateMsg(), Sequencer::Sequencer(), ComputePmeCUDAMgr::setupPencils(), ComputePmeMgr::ungridCalc(), and ComputeCUDAMgr::update().

deviceIDList

int deviceIDList[MAX_NUM_RANKS]

Definition at line 92 of file DeviceCUDA.C.

Referenced by DeviceCUDA::getDeviceIDforPe(), and DeviceCUDA::initialize().

masterPeList

int masterPeList[MAX_NUM_DEVICES]

Definition at line 95 of file DeviceCUDA.C.

Referenced by DeviceCUDA::getMasterPeForDeviceID(), DeviceCUDA::initialize(), and SynchronousCollectives::initMasterScope().