NAMD
BackEnd.C
Go to the documentation of this file.
1 
7 #include "BackEnd.h"
8 #include "ProcessorPrivate.h"
9 #include "common.h"
10 #include "Node.h"
11 #include "memusage.h"
12 
13 #include <new>
14 #if defined(WIN32) && !defined(__CYGWIN__)
15 #include <new.h>
16 #endif
17 
18 #include "Lattice.h"
19 #include "ComputeMoa.h"
20 #include "ComputeMsmMsa.h" // needed for MsmMsaData definition
21 #include "main.decl.h"
22 #include "main.h"
23 #include "BOCgroup.h"
24 #include "WorkDistrib.decl.h"
25 #include "ProxyMgr.decl.h"
26 #include "PatchMgr.decl.h"
27 #include "DataExchanger.decl.h"
28 #ifdef CHARM_HAS_MSA
29 #include "ComputeMgr.decl.h"
30 #endif
31 #include "ReductionMgr.decl.h"
32 #include "CollectionMgr.decl.h"
33 #include "CollectionMaster.decl.h"
34 #include "CollectionMgr.h"
35 #include "CollectionMaster.h"
36 #include "BroadcastMgr.decl.h"
37 #include "LdbCoordinator.decl.h"
38 #include "Sync.decl.h"
39 
40 #ifdef MEM_OPT_VERSION
41 #include "ParallelIOMgr.decl.h"
42 #endif
43 
44 #ifdef NAMD_TCL
45 #include <tcl.h>
46 #endif
47 
48 extern void _initCharm(int, char**);
49 
52 
53 CkpvStaticDeclare(int,exitSchedHndlr);
54 
55 extern "C" void exit_sched(void* msg)
56 {
57  // CmiPrintf("Exiting scheduler on %d\n",CmiMyPe());
58  CsdExitScheduler();
59 }
60 
61 static void register_exit_sched(void)
62 {
63  CkpvInitialize(int,exitSchedHndlr);
64  CkpvAccess(exitSchedHndlr) = CmiRegisterHandler((CmiHandler)exit_sched);
65 }
66 
68 {
69  void* msg = CmiAlloc(CmiMsgHeaderSizeBytes);
70  CmiSetHandler(msg,CkpvAccess(exitSchedHndlr));
71  CmiSyncSendAndFree(pe,CmiMsgHeaderSizeBytes,(char *)msg);
72 }
73 
74 #if defined(WIN32) && !defined(__CYGWIN__) && !defined(__MINGW_H)
75 int NAMD_new_handler(size_t) {
76 #else
78 #endif
79  char tmp[100];
80  sprintf(tmp,"Memory allocation failed on processor %d.",CmiMyPe());
81  NAMD_die(tmp);
82 #if defined(WIN32) && !defined(__CYGWIN__) && !defined(__MINGW_H)
83  return 0;
84 #endif
85 }
86 
87 void topo_getargs(char**);
88 void cuda_getargs(char**);
89 // void cuda_initialize();
91 void mic_getargs(char**);
92 // void mic_initialize();
93 #ifdef NAMD_AVXTILES
94 // For "+notiles" commandline option to disable "Tiles" algorithm
95 int avxTilesCommandLineDisable = 0;
96 #endif
97 
98 // called on all procs
99 void all_init(int argc, char **argv)
100 {
101 #if defined(WIN32) && !defined(__CYGWIN__) && !defined(__MINGW_H)
102  _set_new_handler(NAMD_new_handler);
103 #else
104  std::set_new_handler(NAMD_new_handler);
105 #endif
108  CmiGetArgFlag(argv, "+idlepoll"); // remove +idlepoll if it's still there
109  topo_getargs(argv);
110 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
111  cuda_getargs(argv);
112  argc = CmiGetArgc(argv);
113 #endif
114 #ifdef NAMD_MIC
115  CmiGetArgFlag(argv, "+idlepoll"); // remove +idlepoll if it's still there
116  mic_getargs(argv);
117  argc = CmiGetArgc(argv);
118 #endif
119 #ifdef NAMD_AVXTILES
120  avxTilesCommandLineDisable = CmiGetArgFlag(argv, "+notiles");
121 #endif
122 
123 #ifdef NAMD_CUDA
124  // launch CUDA runtime threads before affinity is set by _initCharm
125  // _initCharm contains a node barrier so there is no race here
126  if(CmiMyRank()==0){
128  }
129 #endif
130 
131  _initCharm(argc, argv); // message main Chare
132 
133 //#if 0 // moved to WorkDistrib
134 //#ifdef NAMD_CUDA
135 // if ( CkMyPe() < CkNumPes() ) cuda_initialize();
136 //#endif
137 //#ifdef NAMD_MIC
138 // if ( CkMyPe() < CkNumPes() ) mic_initialize();
139 //#endif
140 //#endif
141 }
142 
143 extern void after_backend_init(int argc, char **argv);
144 void master_init(int argc, char **argv);
145 
146 // called on slave procs
147 void slave_init(int argc, char **argv)
148 {
149 #if CMK_SMP
150  //the original main thread could now be a comm thread
151  //and a slave thread could now be the main thread,
152  //so we have to do the master initialization here
153  if(CmiMyRank()==0){
154  master_init(argc, argv);
155  if(CmiMyPe()==0)
156  after_backend_init(argc, argv);
157  return;
158  }
159 #endif
160 
161  all_init(argc, argv);
162 
163  if (CkMyRank() < CkMyNodeSize()) // skip the communication thread
164  CsdScheduler(-1);
165 }
166 
167 void master_init(int argc, char **argv){
168  cpuTime_start = CmiCpuTimer();
169  wallTime_start = CmiWallTimer();
170  if ( CmiMyPe() ) {
171  all_init(argc, argv);
172  CsdScheduler(-1);
173  ConverseExit(); // should never return
174  }
175 
176  all_init(argc, argv);
177 
178  // Create branch-office chares
179  BOCgroup group;
180  group.workDistrib = CProxy_WorkDistrib::ckNew();
181  group.proxyMgr = CProxy_ProxyMgr::ckNew();
182  group.patchMgr = CProxy_PatchMgr::ckNew();
183  group.computeMgr = CProxy_ComputeMgr::ckNew();
184  group.reductionMgr = CProxy_ReductionMgr::ckNew();
185  // group.computePmeMgr set in constructor during startup
186  group.nodePmeMgr = CProxy_NodePmeMgr::ckNew();
187 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
188  group.computePmeCUDAMgr = CProxy_ComputePmeCUDAMgr::ckNew();
189  group.computeCUDAMgr = CProxy_ComputeCUDAMgr::ckNew();
190 #endif
191 #ifdef OPENATOM_VERSION
192  group.computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
193 #endif // OPENATOM_VERSION
194  group.computeExtMgr = CProxy_ComputeExtMgr::ckNew();
195  group.computeQMMgr = CProxy_ComputeQMMgr::ckNew();
196  group.computeGBISserMgr = CProxy_ComputeGBISserMgr::ckNew();
197  group.computeFmmSerialMgr = CProxy_ComputeFmmSerialMgr::ckNew();
198  group.computeMsmSerialMgr = CProxy_ComputeMsmSerialMgr::ckNew();
199 #ifdef CHARM_HAS_MSA
200  group.computeMsmMsaMgr = CProxy_ComputeMsmMsaMgr::ckNew();
201 #endif
202  group.computeMsmMgr = CProxy_ComputeMsmMgr::ckNew();
203  // Charm CkMulticast library module
204  group.multicastMgr = CProxy_CkMulticastMgr::ckNew();
205 #ifdef MEM_OPT_VERSION
206  group.ioMgr=CProxy_ParallelIOMgr::ckNew();
207 #endif
208 
209  group.sync = CProxy_Sync::ckNew();
210 
211  #ifdef USE_NODEPATCHMGR
212  group.nodeProxyMgr = CProxy_NodeProxyMgr::ckNew();
213  #endif
214 
215 #if CMK_SMP && USE_CKLOOP
216  group.ckLoop = CkLoop_Init();
217 #endif
218 
219  CkChareID collectionMaster = CProxy_CollectionMaster::ckNew(0);
220  SlaveInitMsg *initmsg7 = new SlaveInitMsg;
221  initmsg7->master = collectionMaster;
222  group.collectionMgr = CProxy_CollectionMgr::ckNew(initmsg7);
223 
224  group.broadcastMgr = CProxy_BroadcastMgr::ckNew();
225  group.ldbCoordinator = CProxy_LdbCoordinator::ckNew();
226 
227  group.dataExchanger = CProxy_DataExchanger::ckNew();
228 
229  GroupInitMsg *msg = new GroupInitMsg;
230  msg->group = group;
231  CkGroupID node = CProxy_Node::ckNew(msg);
232 
233  CkStartQD(CkCallback(CkIndex_Node::exitScheduler(), CkMyPe(), node));
234  CsdScheduler(-1);
235 }
236 
237 char *gNAMDBinaryName = NULL;
238 // called by main on one or all procs
239 void BackEnd::init(int argc, char **argv) {
240 
241  gNAMDBinaryName = argv[0]+strlen(argv[0])-1;
242  while(gNAMDBinaryName != argv[0]){
243  if(*gNAMDBinaryName=='/' || *gNAMDBinaryName=='\\'){
244  gNAMDBinaryName++;
245  break;
246  }
247  gNAMDBinaryName--;
248  }
249 
250 #if defined(NAMD_CUDA) || defined(NAMD_HIP) || defined(NAMD_MIC)
251  // look for but don't remove +idlepoll on command line
252  int idlepoll = 0;
253  for ( int i = 0; i < argc; ++i ) {
254  if ( 0==strcmp(argv[i],"+idlepoll") ) {
255  idlepoll = 1;
256  break;
257  }
258  }
259 #endif
260 
261  ConverseInit(argc, argv, slave_init, 1, 1); // calls slave_init on others
262 
263 // idlepoll only matters for non-smp UDP layer
264 #if (defined(NAMD_CUDA) || defined(NAMD_HIP) || defined(NAMD_MIC)) && CMK_NET_VERSION && CMK_SHARED_VARS_UNAVAILABLE && CMK_WHEN_PROCESSOR_IDLE_USLEEP && ! CMK_USE_IBVERBS && ! CMK_USE_TCP
265  if ( ! idlepoll ) {
266  NAMD_die("Please add +idlepoll to command line for proper performance.");
267  }
268 #endif
269 
270  master_init(argc, argv);
271 }
272 
273 void cuda_finalize();
274 
275 // called on proc 0 by front end
276 void BackEnd::exit(int status) {
277  float cpuTime = CmiCpuTimer() - cpuTime_start;
278  float wallTime = CmiWallTimer() - wallTime_start;
279  CmiPrintf("====================================================\n\n"
280  "WallClock: %f CPUTime: %f Memory: %f MB\n",
281  wallTime, cpuTime, memusage_MB());
282 #ifdef NAMD_TCL
283  Tcl_Finalize();
284 #endif
285 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
286  cuda_finalize();
287 #ifdef __APPLE__
288 #if 0 && CMK_MULTICORE
289  CmiPrintf("EXITING ABNORMALLY TO AVOID HANGING CUDA RUNTIME THREADS\n");
290  ::exit(0);
291 #endif
292 #endif
293 #endif
294 #ifdef NAMD_MIC
295 #if 0 && CMK_MULTICORE
296  CmiPrintf("EXITING ABNORMALLY TO AVOID HANGING MIC OFFLOAD THREADS\n");
297 #pragma offload target(mic)
298  {
299  ::exit(0);
300  }
301 #endif
302 #endif
303 #if CHARM_VERSION < 61000
304  CkExit();
305 #else
306  CkExit(status);
307 #endif
308 }
309 
310 // start scheduler
311 void BackEnd::suspend(void) {
312  CsdScheduler(-1);
313 }
314 
315 // start quiescence detection to return to front end
316 void BackEnd::awaken(void) {
318 }
319 
320 // start QD and scheduler
321 void BackEnd::barrier(void) {
322  awaken();
323  suspend();
324 }
325 
static Node * Object()
Definition: Node.h:86
CkGroupID broadcastMgr
Definition: BOCgroup.h:42
CkChareID master
Definition: CollectionMgr.h:23
static void awaken(void)
Definition: BackEnd.C:316
static void ExitSchedOn(int pe)
Definition: BackEnd.C:67
void after_backend_init(int argc, char **argv)
Definition: mainfunc.C:54
BOCgroup group
Definition: Node.h:68
CkGroupID patchMgr
Definition: BOCgroup.h:19
void all_init(int argc, char **argv)
Definition: BackEnd.C:99
void NAMD_new_handler()
Definition: BackEnd.C:77
void cuda_getargs(char **)
Definition: DeviceCUDA.C:55
static void exit(int status=0)
Definition: BackEnd.C:276
CkGroupID computeMsmMsaMgr
Definition: BOCgroup.h:37
static void suspend(void)
Definition: BackEnd.C:311
CkGroupID workDistrib
Definition: BOCgroup.h:18
char * gNAMDBinaryName
Definition: BackEnd.C:237
static void init(int argc, char **argv)
Definition: BackEnd.C:239
void master_init(int argc, char **argv)
Definition: BackEnd.C:167
float cpuTime_start
Definition: BackEnd.C:50
CkGroupID computeMgr
Definition: BOCgroup.h:21
CkGroupID computeCUDAMgr
Definition: BOCgroup.h:26
CkGroupID computePmeCUDAMgr
Definition: BOCgroup.h:25
CkGroupID computeMsmSerialMgr
Definition: BOCgroup.h:36
float wallTime_start
Definition: BackEnd.C:51
double memusage_MB()
Definition: memusage.h:13
CkGroupID dataExchanger
Definition: BOCgroup.h:55
CkGroupID collectionMgr
Definition: BOCgroup.h:41
CkGroupID computeMsmMgr
Definition: BOCgroup.h:38
CkGroupID computeExtMgr
Definition: BOCgroup.h:32
CkGroupID multicastMgr
Definition: BOCgroup.h:39
void enableExitScheduler(void)
Definition: Node.C:1343
void ProcessorPrivateInit(void)
CkGroupID proxyMgr
Definition: BOCgroup.h:20
CkGroupID ldbCoordinator
Definition: BOCgroup.h:43
CkGroupID nodePmeMgr
Definition: BOCgroup.h:23
CkGroupID computeGBISserMgr
Definition: BOCgroup.h:34
void NAMD_die(const char *err_msg)
Definition: common.C:85
void exit_sched(void *msg)
Definition: BackEnd.C:55
void mic_getargs(char **)
CkGroupID ioMgr
Definition: BOCgroup.h:46
void topo_getargs(char **)
Definition: WorkDistrib.C:90
static void barrier(void)
Definition: BackEnd.C:321
void _initCharm(int, char **)
static void register_exit_sched(void)
Definition: BackEnd.C:61
CkGroupID sync
Definition: BOCgroup.h:44
void slave_init(int argc, char **argv)
Definition: BackEnd.C:147
CkGroupID computeFmmSerialMgr
Definition: BOCgroup.h:35
void cuda_affinity_initialize()
Definition: CudaUtils.C:33
CkpvStaticDeclare(int, exitSchedHndlr)
CkGroupID computeQMMgr
Definition: BOCgroup.h:33
CkGroupID reductionMgr
Definition: BOCgroup.h:40
void cuda_finalize()
Definition: DeviceCUDA.C:30