NAMD
GlobalGPUMgr.C
Go to the documentation of this file.
1 #include "CudaRecord.h"
2 #include "CudaUtils.h"
3 #include "GlobalGPUMgr.decl.h"
4 #include "GlobalGPUMgr.h"
6 #include "Node.h"
7 #include "SimParameters.h"
8 #include "NamdEventsProfiling.h"
9 
10 #include "charm++.h"
11 
12 #include <sched.h>
13 #include <pthread.h>
14 
15 #include "DeviceCUDA.h"
16 
17 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
18 extern __thread DeviceCUDA *deviceCUDA;
19 
20 RunWithAffinity::RunWithAffinity(const bool valid, const int tempCpu) {
21  validCpu = valid && (tempCpu >= 0) && (tempCpu < CmiNumCores());
22 
23  if (validCpu) {
24  current = CPU_ALLOC(CmiNumCores());
25  temp = CPU_ALLOC(CmiNumCores());
26  size = CPU_ALLOC_SIZE(CmiNumCores());
27 
28  CPU_ZERO_S(size, current);
29  CPU_ZERO_S(size, temp);
30 
31  pthread_getaffinity_np(pthread_self(), size, current);
32 
33  CPU_SET_S(tempCpu, size, temp);
34  pthread_setaffinity_np(pthread_self(), size, temp);
35  }
36 }
37 
39  if (validCpu) {
40  pthread_setaffinity_np(pthread_self(), size, current);
41 
42  CPU_FREE(current);
43  CPU_FREE(temp);
44  }
45 }
46 
48 {
49  if (CkpvAccess(GlobalGPUMgr_instance) == NULL) {
50  CkpvAccess(GlobalGPUMgr_instance) = this;
51  } else {
52  NAMD_bug("GlobalGPUMgr instanced twice on same processor!");
53  }
54 }
55 
57 
60 
61  isMasterPe = (CkMyPe() == deviceCUDA->getMasterPe());
62  isMasterDevice = deviceCUDA->getIsMasterDevice();
63 
64  peToDeviceIDMap.resize(CkNumPes(), -1);
65  peToDeviceIndexMap.resize(CkNumPes(), -1);
66  syncColl->forceBarrierAll(); //Make sure maps are resized
67 
68  const int myDeviceId = (isMasterPe) ? deviceCUDA->getDeviceID() : -1;
69  peToDeviceIDMap = syncColl->allGather<int>(myDeviceId, SynchronousCollectiveScope::all);
70 
71  // Count number of devices first
72  numDevices = 0;
73  for (int i = 0; i < CkNumPes(); i++) {
74  if (peToDeviceIDMap[i] != -1) {
75  numDevices++;
76  }
77  }
78 
79  deviceIndexToPeMap.resize(numDevices, 0);
80 
81  // Setup maps
82  int current_device = 0;
83  for (int i = 0; i < CkNumPes(); i++) {
84  if (peToDeviceIDMap[i] != -1) {
85  deviceIndexToPeMap[current_device] = i;
86  peToDeviceIndexMap[i] = current_device;
87  current_device++;
88  }
89  }
90 
91  deviceIndex = peToDeviceIndexMap[CkMyPe()];
92 }
93 
95 }
96 
98 }
99 
101  return deviceCUDA->getPmeDeviceIndex() == deviceIndex;
102 }
103 
104 #endif /* NAMD_CUDA || NAMD_HIP */
105 
106 #include "GlobalGPUMgr.def.h"
107 
std::vector< T > allGather(const T &data, const SynchronousCollectiveScope scope)
__thread DeviceCUDA * deviceCUDA
Definition: DeviceCUDA.C:23
void initialize()
Definition: GlobalGPUMgr.C:58
int getPmeDeviceIndex()
Definition: DeviceCUDA.h:167
int getMasterPe()
Definition: DeviceCUDA.h:137
void NAMD_bug(const char *err_msg)
Definition: common.C:195
void finalize()
Definition: GlobalGPUMgr.C:97
void initializeBackends()
Definition: GlobalGPUMgr.C:94
int getDeviceID()
Definition: DeviceCUDA.h:144
RunWithAffinity(const bool valid, const int tempCpu)
Definition: GlobalGPUMgr.C:20
bool getIsMasterDevice()
Definition: DeviceCUDA.C:642
bool getIsPmeDevice()
Definition: GlobalGPUMgr.C:100
static SynchronousCollectives * Object()