NAMD
Classes | Macros | Functions | Variables
DeviceCUDA.C File Reference
#include <cuda_runtime.h>
#include <cuda.h>
#include "common.h"
#include "charm++.h"
#include "DeviceCUDA.h"
#include "WorkDistrib.h"
#include "CudaUtils.h"
#include "Node.h"
#include "SimParameters.h"

Go to the source code of this file.

Classes

struct  cuda_args_t
 

Macros

#define MAX_NUM_RANKS   2048
 
#define MAX_NUM_DEVICES   256
 
#define REGISTER_DEVICE_EVENTS(DEV)
 

Functions

void cuda_finalize ()
 
void cuda_initialize ()
 
void cuda_getargs (char **argv)
 

Variables

__thread DeviceCUDAdeviceCUDA
 
static __thread cuda_args_t cuda_args
 
int deviceIDList [MAX_NUM_RANKS]
 
int masterPeList [MAX_NUM_DEVICES]
 

Macro Definition Documentation

◆ MAX_NUM_DEVICES

#define MAX_NUM_DEVICES   256

Definition at line 92 of file DeviceCUDA.C.

Referenced by DeviceCUDA::initialize().

◆ MAX_NUM_RANKS

#define MAX_NUM_RANKS   2048

Definition at line 89 of file DeviceCUDA.C.

Referenced by DeviceCUDA::initialize().

◆ REGISTER_DEVICE_EVENTS

#define REGISTER_DEVICE_EVENTS (   DEV)
Value:
traceRegisterUserEvent("CUDA device " #DEV " remote", CUDA_EVENT_ID_BASE + 2 * DEV); \
traceRegisterUserEvent("CUDA device " #DEV " local", CUDA_EVENT_ID_BASE + 2 * DEV + 1);
#define CUDA_EVENT_ID_BASE
Definition: DeviceCUDA.h:42

Function Documentation

◆ cuda_finalize()

void cuda_finalize ( )

Definition at line 34 of file DeviceCUDA.C.

Referenced by cuda_initialize().

34  {
35 #if defined(NAMD_CUDA) || ((NAMD_HIP) && ((HIP_VERSION_MAJOR < 4) && (HIP_VERSION_MINOR < 5)))
36  int ndevs = 0;
37  cudaGetDeviceCount(&ndevs);
38  for ( int dev=0; dev < ndevs; ++dev ) {
39  cudaSetDevice(dev);
40  cudaDeviceReset();
41  }
42 #else
43  // for hip on rocm versions later > 4.5, there seems to be
44  // additional hipFree's called after hipDeviceReset, so removing it for now
45 #endif
46 }

◆ cuda_getargs()

void cuda_getargs ( char **  argv)

Definition at line 67 of file DeviceCUDA.C.

References cuda_args, cuda_args_t::devicelist, cuda_args_t::devicesperreplica, cuda_args_t::globaldevice, cuda_args_t::ignoresharing, cuda_args_t::mergegrids, NAMD_die(), cuda_args_t::nomergegrids, cuda_args_t::nostreaming, cuda_args_t::pmedevice, cuda_args_t::pmePes, and cuda_args_t::usedevicelist.

Referenced by all_init().

67  {
69  cuda_args.usedevicelist = CmiGetArgStringDesc(argv, "+devices", &cuda_args.devicelist,
70  "comma-delimited list of CUDA device numbers such as 0,2,1,2");
72  CmiGetArgInt(argv, "+devicesperreplica", &cuda_args.devicesperreplica);
73  if ( cuda_args.devicesperreplica < 0 ) NAMD_die("Devices per replica must be positive\n");
74  cuda_args.ignoresharing = CmiGetArgFlag(argv, "+ignoresharing");
75  cuda_args.mergegrids = CmiGetArgFlag(argv, "+mergegrids");
76  cuda_args.nomergegrids = CmiGetArgFlag(argv, "+nomergegrids");
77  if ( cuda_args.mergegrids && cuda_args.nomergegrids ) NAMD_die("Do not specify both +mergegrids and +nomergegrids");
78  cuda_args.nostreaming = CmiGetArgFlag(argv, "+nostreaming");
79 #ifdef NODEGROUP_FORCE_REGISTER
80  CmiGetArgInt(argv, "+pmedevice", &cuda_args.pmedevice);
81  CmiGetArgInt(argv, "+pmepes", &cuda_args.pmePes);
83  CmiGetArgInt(argv, "+globaldevice", &cuda_args.globaldevice);
84 #endif
85 }
int devicesperreplica
Definition: DeviceCUDA.C:55
int nomergegrids
Definition: DeviceCUDA.C:58
static __thread cuda_args_t cuda_args
Definition: DeviceCUDA.C:65
int globaldevice
Definition: DeviceCUDA.C:62
int usedevicelist
Definition: DeviceCUDA.C:54
int mergegrids
Definition: DeviceCUDA.C:57
int nostreaming
Definition: DeviceCUDA.C:59
int pmedevice
Definition: DeviceCUDA.C:60
void NAMD_die(const char *err_msg)
Definition: common.C:147
int ignoresharing
Definition: DeviceCUDA.C:56
char * devicelist
Definition: DeviceCUDA.C:53

◆ cuda_initialize()

void cuda_initialize ( )

Definition at line 27 of file DeviceCUDA.C.

References cuda_finalize(), deviceCUDA, and DeviceCUDA::initialize().

Referenced by WorkDistrib::peOrderingReady().

27  {
28  deviceCUDA = new DeviceCUDA();
30  std::atexit(cuda_finalize);
31 }
void initialize()
Definition: DeviceCUDA.C:107
void cuda_finalize()
Definition: DeviceCUDA.C:34
__thread DeviceCUDA * deviceCUDA
Definition: DeviceCUDA.C:23

Variable Documentation

◆ cuda_args

__thread cuda_args_t cuda_args
static

Definition at line 65 of file DeviceCUDA.C.

Referenced by cuda_getargs(), and DeviceCUDA::initialize().

◆ deviceCUDA

__thread DeviceCUDA* deviceCUDA

◆ deviceIDList

int deviceIDList[MAX_NUM_RANKS]

Definition at line 90 of file DeviceCUDA.C.

Referenced by DeviceCUDA::getDeviceIDforPe(), and DeviceCUDA::initialize().

◆ masterPeList

int masterPeList[MAX_NUM_DEVICES]

Definition at line 93 of file DeviceCUDA.C.

Referenced by DeviceCUDA::getMasterPeForDeviceID(), and DeviceCUDA::initialize().