NAMD
Node.C
Go to the documentation of this file.
1 
7 /*
8  Toplevel routines for initializing a Node for a simulation
9  one Node per Pe (processor element).
10 */
11 
12 #if !defined(WIN32) || defined(__CYGWIN__)
13 #include <unistd.h>
14 #endif
15 #include "InfoStream.h"
16 #include "Node.decl.h"
17 #include "Node.h"
18 #ifdef DPMTA
19 #include <pvm3.h>
20 #endif
21 
22 #include "ProcessorPrivate.h"
23 
24 #define MIN_DEBUG_LEVEL 3
25 //#define DEBUGM
26 #include "Debug.h"
27 
28 #include <stdio.h>
29 #include <converse.h>
30 #include "memusage.h"
31 #include "IMDOutput.h"
32 #include "Lattice.h"
33 #include "ComputeMsmMsa.h" // needed for MsmMsaData definition
34 #include "ComputeMsm.h" // needed for MsmInitMsg definition
35 #include "main.decl.h"
36 #include "main.h"
37 #include "WorkDistrib.h"
38 #include "PatchMgr.h"
39 #include "Patch.h"
40 #include "Compute.h"
41 #include "ComputeMap.h"
42 #include "ComputeMgr.h"
43 #include "Molecule.h"
44 #include "HomePatchList.h"
45 #include "AtomMap.h"
46 #include "Sequencer.h"
47 #include "Controller.h"
48 #include "NamdState.h"
49 #include "Output.h"
50 #include "ProxyMgr.h"
51 #include "PatchMap.h"
52 #include "PatchMap.inl"
53 #include "Parameters.h"
54 #include "SimParameters.h"
55 #include "Communicate.h"
56 #include "LdbCoordinator.h"
57 #include "ScriptTcl.h"
58 #include "ComputeMgr.decl.h"
59 #include "ComputePmeMgr.decl.h"
60 // #ifdef NAMD_CUDA
61 #include "ComputeCUDAMgr.decl.h"
62 #include "ComputeCUDAMgr.h"
63 #include "ComputePmeCUDAMgr.decl.h"
64 #include "ComputePmeCUDAMgr.h"
65 // #endif
66 #include "ComputeGridForceMgr.decl.h"
67 #include "Sync.h"
68 #include "BackEnd.h"
69 #include "PDB.h"
70 #include "packmsg.h"
71 #include "CollectionMgr.decl.h"
72 #include "ParallelIOMgr.decl.h"
73 #include "Vector.h"
74 // BEGIN LA
75 #include "Random.h"
76 // END LA
77 #include "PatchData.h"
78 #include "DeviceCUDA.h"
79 
80 #include "NamdEventsProfiling.h"
81 
82 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
83 extern "C" void CApplicationInit();
84 #endif
85 
86 #include "DumpBench.h"
87 
88 #ifdef NAMD_CUDA
89 extern __thread DeviceCUDA *deviceCUDA;
90 #endif
91 
92 class CheckpointMsg : public CMessage_CheckpointMsg {
93 public:
94  int task;
95  int replica;
97  char *key;
98 };
99 
100 extern "C" {
101  void recvCheckpointCReq_handler(envelope*);
102  void recvCheckpointCAck_handler(envelope*);
103 }
104 
105 #if defined(CMK_BALANCED_INJECTION_API) && CMK_BALANCED_INJECTION_API != 0
106 #include "ckBIconfig.h"
107 #endif
108 
109 #include "CollectionMgr.h"
110 #include "CollectionMaster.h"
111 #include "CollectionMgr.decl.h"
112 #include "CollectionMaster.decl.h"
113 
114 #if USE_HPM
115 extern "C" void HPM_Init(int);
116 extern "C" void HPM_Start(char *label, int);
117 extern "C" void HPM_Stop(char *label, int);
118 extern "C" void HPM_Print(int, int);
119 #endif
120 
121 #if defined(NAMD_MIC)
122  extern void mic_dumpHostDeviceComputeMap();
123  extern void mic_initHostDeviceLDB();
124 #endif
125 
126 #ifdef MEASURE_NAMD_WITH_PAPI
127 #include "papi.h"
128 #if CMK_SMP
129 #include <pthread.h>
130 #endif
131 #define NUM_PAPI_EVENTS 6
132 CkpvDeclare(int *, papiEvents);
133 
134 #define MEASURE_PAPI_SPP 1
135 #define MEASURE_PAPI_CACHE 0
136 #define MEASURE_PAPI_FLOPS 0
137 
138 static void namdInitPapiCounters(){
139  if(CkMyRank()==0){
140  //only initialize per OS process (i.e. a charm node)
141  int retval = PAPI_library_init(PAPI_VER_CURRENT);
142  if(retval != PAPI_VER_CURRENT) {
143  if(CkMyPe()==0){
144  NAMD_die("PAPI library is not compatitible!");
145  }
146  }
147  #if CMK_SMP
148  //now only consider systems that are compatible with POSIX
149  if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
150  if(CkMyPe()==0){
151  NAMD_die("Multi-thread mode in PAPI could not be initialized!");
152  }
153  }
154  #endif
155  }
156  CkpvInitialize(int *, papiEvents);
157  CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS+1];
158 
159 #if MEASURE_PAPI_CACHE
160  if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
161  CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
162  }else{
163  if(CkMyPe()==0){
164  CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");
165  }
166  //if not default to PAPI_TOT_INS
167  CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
168  }
169 
170  if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
171  CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
172  }else{
173  //if not default to PAPI_TOT_CYC
174  CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
175  }
176 #elif MEASURE_PAPI_FLOPS
177  if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
178  CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
179  }else{
180  if(CkMyPe()==0){
181  CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
182  }
183  //if not default to PAPI_TOT_INS
184  CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
185  }
186 
187  if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
188  CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
189  }else{
190  //if not default to PAPI_TOT_CYC
191  CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
192  }
193 #elif MEASURE_PAPI_SPP
194 /* for SPP we record these
195 1) PAPI_FP_OPS
196 2) PAPI_TOT_INS
197 3) perf::PERF_COUNT_HW_CACHE_LL:MISS
198 4) DATA_PREFETCHER:ALL
199 5) PAPI_L1_DCA
200 6) INSTRUCTION_FETCH_STALL
201 7) PAPI_TOT_CYC, and
202 8) real (wall) time
203 */
204  int papiEventSet = PAPI_NULL;
205  if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
206  CmiAbort("PAPI failed to create event set!\n");
207  }
208 
209  if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
210  CkpvAccess(papiEvents)[0] = PAPI_FP_OPS;
211  }else{
212  if(CkMyPe()==0){
213  CkAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
214  }
215  }
216  if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
217  CkpvAccess(papiEvents)[1] = PAPI_TOT_INS;
218  }else{
219  if(CkMyPe()==0){
220  CkAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
221  }
222  }
223  int EventCode;
224  int ret;
225  ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
226  if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
227  CkpvAccess(papiEvents)[2] = EventCode;
228  }else{
229  if(CkMyPe()==0){
230  CkAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
231  }
232  }
233  ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
234  if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
235  CkpvAccess(papiEvents)[3] = EventCode;
236  }else{
237  if(CkMyPe()==0){
238  CkAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
239  }
240  }
241  if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
242  CkpvAccess(papiEvents)[4] = PAPI_L1_DCA;
243  }else{
244  if(CkMyPe()==0){
245  CkAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
246  }
247  }
248  /* ret=PAPI_event_name_to_code("INSTRUCTION_FETCH_STALL",&EventCode);
249  if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
250  CkpvAccess(papiEvents)[5] = EventCode;
251  }else{
252  if(CkMyPe()==0){
253  CkAbort("WARNING: INSTRUCTION_FETCH_STALL doesn't exist on this platform!");
254  }
255  }
256  */
257  if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
258  CkpvAccess(papiEvents)[5] = PAPI_TOT_CYC;
259  }else{
260  if(CkMyPe()==0){
261  CkAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
262  }
263  }
264  for(int i=0;i<NUM_PAPI_EVENTS;i++)
265  {
266  int papiRetValue=PAPI_add_events(papiEventSet, &CkpvAccess(papiEvents)[i],1);
267  if (papiRetValue != PAPI_OK) {
268  CkPrintf("failure for event %d\n",i);
269  if (papiRetValue == PAPI_ECNFLCT) {
270  CmiAbort("PAPI events conflict! Please re-assign event types!\n");
271  } else {
272  CmiAbort("PAPI failed to add designated events!\n");
273  }
274  }
275 
276  }
277 #endif
278 }
279 #endif
280 
281 #ifdef OPENATOM_VERSION
282 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
283 {
284  CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
285 }
286 #endif //OPENATOM_VERSION
287 
288 //======================================================================
289 // Public Functions
290 
291 //----------------------------------------------------------------------
292 
294 double startupTime;
295 
296 //----------------------------------------------------------------------
297 // BOC constructor
299 {
300  DebugM(4,"Creating Node\n");
301 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
302  CApplicationInit();
303 #endif
304  if (CkpvAccess(Node_instance) == 0) {
305  CkpvAccess(Node_instance) = this;
306  eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep", 135);
307  } else {
308  NAMD_bug("Node::Node() - another instance of Node exists!");
309  }
310 
311  CkpvAccess(BOCclass_group) = msg->group;
312  delete msg;
313 
314  CkpvAccess(BOCclass_group).node = thisgroup;
315 
316  recvCheckpointCReq_index = CmiRegisterHandler((CmiHandler)recvCheckpointCReq_handler);
317  recvCheckpointCAck_index = CmiRegisterHandler((CmiHandler)recvCheckpointCAck_handler);
318 
319  startupPhase = 0;
320 
321  molecule = NULL;
322  parameters = NULL;
323  simParameters = NULL;
324  configList = NULL;
325  pdb = NULL;
326  state = NULL;
327  output = NULL;
328  imd = new IMDOutput;
329  colvars = 0;
330 
331 #if USE_HPM
332  // assumes that this will be done only on BG/P
333  TopoManager *tmgr = new TopoManager();
334  int x, y, z;
335  tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
336  delete tmgr;
337 #endif
338 
339  specialTracing = traceAvailable() && (traceIsOn()==0);
340 
341  DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
344  if ( CkMyRank() == 0 ) ComputeMap::Instance();
345 
346  //Note: Binding BOC vars such as workDistrib has been moved
347  //to the 1st phase of startup because the in-order message delivery
348  //is not always guaranteed --Chao Mei
349 #if defined(CMK_BALANCED_INJECTION_API) && CMK_BALANCED_INJECTION_API != 0
350  if(CkMyRank() == 0){
351  balancedInjectionLevel=ck_get_GNI_BIConfig();
352  // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
353  ck_set_GNI_BIConfig(20);
354  // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
355  }
356 #endif
357 
358 }
359 
360 //----------------------------------------------------------------------
361 // ~Node(void) needs to clean up everything.
362 
364 {
365  delete imd;
366  delete output;
367  delete computeMap;
368  delete atomMap;
369  delete patchMap;
370  delete CkpvAccess(comm);
371  // BEGIN LA
372  delete rand;
373  // END LA
374 #ifdef MEASURE_NAMD_WITH_PAPI
375  delete CkpvAccess(papiEvents);
376 #endif
377 }
378 
379 void Node::bindBocVars(){
380  DebugM(4,"Binding to BOC's\n");
381  CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
382  patchMgr = pm.ckLocalBranch();
383  CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
384  proxyMgr = prm.ckLocalBranch();
385  CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
386  workDistrib = wd.ckLocalBranch();
387  CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
388  computeMgr = cm.ckLocalBranch();
389  CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
390  ldbCoordinator = lc.ckLocalBranch();
391  #ifdef MEM_OPT_VERSION
392  CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
393  ioMgr = io.ckLocalBranch();
394  #endif
395 
396 }
397 
398 //----------------------------------------------------------------------
399 // Malloc Test Sequence
400 void Node::mallocTest(int step) {
401  int MB = 1024*1024;
402  int size = 100;
403  char* foo = (char*) malloc(size*MB);
404  if ( ! foo ) {
405  char buf[256];
406  sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
407  NAMD_die(buf);
408  }
409  memset(foo,0,size*MB*sizeof(char));
410 }
411 
413  if ( mallocTest_size ) {
414  CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
415  } else {
416  CkPrintf("Starting malloc test on all PEs.\n");
417  }
418  fflush(stdout);
419  ++mallocTest_size;
420  CkStartQD(CkIndex_Node::mallocTestQd(), &thishandle);
421  (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
422 }
423 
424 //----------------------------------------------------------------------
425 // Startup Sequence
426 
428  (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
429 }
430 
434 
435 extern void registerUserEventsForAllComputeObjs(void);
436 
438  NAMD_EVENT_START(1, NamdProfileEvent::NAMD_STARTUP);
439 
440  int gotoRun = false;
441  double newTime;
442 
443  if (!CkMyPe()) {
444  if (!startupPhase) {
445  iout << iINFO << "\n";
446  startupTime = CmiWallTimer();
447  iout << iINFO << "Entering startup at " << startupTime << " s, ";
448  } else {
449  newTime = CmiWallTimer();
450  iout << iINFO << "Startup phase " << startupPhase-1 << " took "
451  << newTime - startupTime << " s, ";
452  startupTime = newTime;
453  }
454  iout << memusage_MB() << " MB of memory in use\n" << endi;
455  fflush(stdout);
456  }
457  switch (startupPhase) {
458 
459  case 0:
461  namdOneCommInit(); // Namd1.X style
462  break;
463 
464  case 1:
465  bindBocVars();
466 
467  // send & receive molecule, simparameters... (Namd1.X style)
468  if (CkMyPe()) {
469  namdOneRecv();
470  } else {
471  namdOneSend();
472  }
473  break;
474 
475  case 2:
476  // fix up one-per-node objects (for SMP version)
480 
483 
484  #if !CMK_SMP || ! USE_CKLOOP
485  //the CkLoop library should be only used in SMP mode
487  #else
488  if ( CkNumPes() < 2 * CkNumNodes() ) simParameters->useCkLoop = 0;
489  #endif
490 
491 
492  if ( simParameters->mallocTest ) {
493  if (!CkMyPe()) {
494  mallocTest_size = 0;
495  CkStartQD(CkIndex_Node::mallocTestQd(), &thishandle);
496  }
497  return;
498  }
499 
500 
501  #ifdef MEASURE_NAMD_WITH_PAPI
502  if(simParameters->papiMeasure) namdInitPapiCounters();
503  #endif
504 
505  #ifdef MEM_OPT_VERSION
506  //At this point, each Node object has received the simParameters,
507  //parameters and the atom signatures info from the master Node
508  //(proc 0). It's time to initialize the parallel IO manager and
509  //read the binary per-atom file --Chao Mei
510 
511  //Step 1: initialize the parallel IO manager per Node
512  ioMgr->initialize(this);
513  #endif
514 
515  break;
516 
517  case 3:
518 
519  #ifdef MEM_OPT_VERSION
520  //Step 2: read the binary per-atom files (signater index, coordinates etc.)
521  ioMgr->readPerAtomInfo();
522  #endif
523 
524  break;
525 
526  case 4:
527 
528  #ifdef MEM_OPT_VERSION
529  //Step 3: update counters of tuples and exclusions inside Molecule object
530  ioMgr->updateMolInfo();
531 
532  //Step 4: prepare distributing the atoms to neighboring procs if necessary
533  ioMgr->migrateAtomsMGrp();
534 
535  //step 5: initialize patchMap and send it to every other processors
536  //to decide atoms to patch distribution on every input processor
537  if(!CkMyPe()) {
538  workDistrib->patchMapInit(); // create space division
540  }
541  #endif
542 
543  #if USE_HPM
544  HPM_Init(localRankOnNode);
545  #endif
546 
547  // take care of inital thread setting
548  threadInit();
549 
550  // create blank AtomMap
552 
553  if (!CkMyPe()) {
554 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
555  if (simParameters->usePMECUDA) {
556  // computePmeCUDAMgr was created in BackEnd.C
557  // This empty branch is to avoid initializing ComputePmeMgr
558  } else
559 #endif
560  if (simParameters->PMEOn) {
561  CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
562  }
563  #ifdef OPENATOM_VERSION
564  if ( simParameters->openatomOn ) {
565  CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
566  }
567  #endif // OPENATOM_VERSION
568 
569  }
570 
571  #ifdef OPENATOM_VERSION
572  if ( simParameters->openatomOn ) {
573  // if ( ! CkMyPe() ) {
574  CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle);
575  startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
576  // }
577  }
578  #endif // OPENATOM_VERSION
579 
580  // BEGIN LA
582  rand->split(CkMyPe(), CkNumPes());
583  // END LA
584 
585  break;
586 
587  case 5:
588  #ifdef MEM_OPT_VERSION
589  //Now, every input proc has received all the atoms necessary
590  //to decide the patches those atoms belong to
591 
592  //step 1: integrate the migrated atoms into the atom list that
593  //contains the initally distributed atoms, and sort the atoms
594  //based on hydrogenList value
595  ioMgr->integrateMigratedAtoms();
596 
597  //step 2: integrate the cluster size of each atom on each output proc
598  ioMgr->integrateClusterSize();
599 
600  //step 3: calculate the number of atoms in each patch on every
601  //input procs (atoms belonging to a patch may lie on different
602  //procs), and reduce such info on proc 0. Such info is required
603  //for determing which node a particular patch is assigned to.
604  ioMgr->calcAtomsInEachPatch();
605 
606  //set to false to re-send PatchMap later
608  #endif
609  break;
610  case 6:
613  }
616  }
619  }
620  #ifdef PROCTRACE_DEBUG
621  DebugFileTrace::Instance("procTrace");
622  #endif
623 
624  if (!CkMyPe()) {
625  output = new Output; // create output object just on PE(0)
626 
627  #ifndef MEM_OPT_VERSION
628  workDistrib->patchMapInit(); // create space division
629  workDistrib->createHomePatches(); // load atoms into HomePatch(es)
630  #endif
631 
634  //ComputeMap::Object()->printComputeMap();
635 
636  // For MIC runs, take the additional step after the compute map has been created to
637  // assign the various computes to either the host or the device. This info will
638  // be distributed across the PEs.
639  #if defined(NAMD_MIC)
640  mic_initHostDeviceLDB();
641  #endif
642 
644  iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
645  << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
646  outputPatchComputeMaps("init_mapping", 0);
647  iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
648  BackEnd::exit();
649  }
650 
652 
653  //in MEM_OPT_VERSION, patchMap is resent
654  //because they have been updated since creation including
655  //#atoms per patch, the proc a patch should stay etc. --Chao Mei
657  #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
658  CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
659  //a node broadcast
660  npm.createProxyInfo(PatchMap::Object()->numPatches());
661  #endif
662  }
663  {
664  #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
665  CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
666  if(CkMyRank()==0) {
667  //just need to register once
668  npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
669  }
670  npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
671  #endif
672  }
673  break;
674 
675  case 7:
676 #ifdef CHARM_HAS_MSA
678  CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
679  msm[CkMyPe()].initialize(new CkQdMsg);
680  }
681 #else
683  CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
684  MsmInitMsg *msg = new MsmInitMsg;
685  Lattice lattice = simParameters->lattice; // system lattice vectors
686  ScaledPosition smin=0, smax=0;
687  if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
688  msg->smin = smin;
689  msg->smax = smax;
690  msm[CkMyPe()].initialize(msg); // call from my own PE
691  }
692  else if ( ! CkMyPe() ) {
693  pdb->get_extremes(smin, smax); // only available on PE 0
694  msg->smin = smin;
695  msg->smax = smax;
696  msm.initialize(msg); // broadcast to chare group
697  }
698 
699  /*
700  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
701  Node *node = nd.ckLocalBranch();
702  ScaledPosition smin, smax;
703  node->pdb->get_extremes(smin, smax);
704  msg->smin = smin; // extreme positions in system
705  msg->smax = smax;
706  msm[CkMyPe()].initialize(msg);
707  */
708  }
709 #endif
710 
711  if ( simParameters->PMEOn ) {
712  #ifdef OPENATOM_VERSION
713  if ( simParameters->openatomOn ) {
714  CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr);
715  moa[CkMyPe()].initialize(new CkQdMsg);
716  }
717  #endif // OPENATOM_VERSION
718 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
719  if ( simParameters->usePMECUDA ) {
720  if(CkMyRank()==0) {
721  CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
722  pme.ckLocalBranch()->initialize(new CkQdMsg); // must run on pe 0 to call ckNew
723  }
724  } else
725 #endif
726  {
727  CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
728  pme[CkMyPe()].initialize(new CkQdMsg);
729  }
730  }
731  break;
732 
733  case 8:
734 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
735  if ( CkMyRank()==0 ) {
736  CProxy_ComputeCUDAMgr nb(CkpvAccess(BOCclass_group).computeCUDAMgr);
737  nb.ckLocalBranch()->initialize(new CkQdMsg);
738  }
739 #endif
740  break;
741 
742  case 9:
744  break;
745 
746  case 10:
747  #ifdef MEM_OPT_VERSION
748  //migrate atoms to HomePatch processors
749  ioMgr->sendAtomsToHomePatchProcs();
750  #endif
751  break;
752 
753  case 11:
754  // part 2 of MSM init
756  CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
757  msm[CkMyPe()].initialize_create(); // call from my own PE
758  }
759 
760  if ( simParameters->PMEOn ) {
761  #ifdef OPENATOM_VERSION
762  if ( simParameters->openatomOn ) {
763  CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr);
764  moa[CkMyPe()].initWorkers(new CkQdMsg);
765  }
766  #endif // OPENATOM_VERSION
767 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
768  if ( simParameters->usePMECUDA ) {
769  if(CkMyRank()==0) {
770  CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
771  pme[CkMyNode()].initialize_pencils(new CkQdMsg);
772  }
773  } else
774 #endif
775  {
776  CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
777  pme[CkMyPe()].initialize_pencils(new CkQdMsg);
778  }
779  }
780 #ifdef CHARM_HAS_MSA
781  else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
782  CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
783  msm[CkMyPe()].initWorkers(new CkQdMsg);
784  }
785 #else
786  else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
787  CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
788  msm[CkMyPe()].update(new CkQdMsg);
789  }
790 #endif
791 
792  #ifdef MEM_OPT_VERSION
793  //Now every processor has all the atoms it needs to create the HomePatches.
794  //The HomePatches are created in parallel on every home patch procs.
795  ioMgr->createHomePatches();
796  #else
797  if (!CkMyPe()) {
799  }
800  #endif
801  break;
802 
803  case 12:
804  if ( simParameters->PMEOn ) {
805  #ifdef OPENATOM_VERSION
806  if ( simParameters->openatomOn ) {
807  CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr);
808  moa[CkMyPe()].startWorkers(new CkQdMsg);
809  }
810  #endif // OPENATOM_VERSION
811 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
812  if ( simParameters->usePMECUDA ) {
813  if(CkMyRank()==0) {
814  CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
815  pme[CkMyNode()].activate_pencils(new CkQdMsg);
816  }
817  } else
818 #endif
819  {
820  CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
821  pme[CkMyPe()].activate_pencils(new CkQdMsg);
822  }
823  }
824 #ifdef CHARM_HAS_MSA
825  else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
826  CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
827  msm[CkMyPe()].startWorkers(new CkQdMsg);
828  }
829 #else
830  /*
831  else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
832  CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
833  //msm[CkMyPe()].startWorkers(new CkQdMsg);
834  }
835  */
836 #endif
837 
838  proxyMgr->createProxies(); // need Home patches before this
839  if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
840 
841 #ifdef NAMD_TCL
842  // TclInitSubsystems() has a race condition so we create one interp per node here
843  if (CkMyPe() && CkMyNodeSize() > 1 && ! CkMyRank()) Tcl_DeleteInterp(Tcl_CreateInterp());
844 #endif
845 
846 #ifdef USE_NODEPATCHMGR
847  //at this point, PatchMap info has been recved on PEs. It is time to create
848  //the home patch spanning tree for receiving proxy list info
849  if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
850  if(CkMyRank()==0) {
851  CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
852  npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
853  }
854  }
855 #endif
856 
857  break;
858 
859  case 13:
860 
861  // DMK - DEBUG - If, in MIC runs, the debug option to dump all the compute maps to files
862  // for debugging/verification purposes has been enabled, have each PE do so now.
863  #if defined(NAMD_MIC)
864  mic_dumpHostDeviceComputeMap();
865  #endif
866 
867  if (!CkMyPe()) {
868  iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
869  << " COMPUTE OBJECTS\n" << endi;
870  }
871  DebugM(4,"Creating Computes\n");
873  DebugM(4,"Building Sequencers\n");
874  buildSequencers();
875  DebugM(4,"Initializing LDB\n");
877  break;
878 
879  case 14:
880  // computes may create proxies on the fly so put these in separate phase
881  Sync::Object()->openSync(); // decide if to open local Sync
883 #if defined(CMK_BALANCED_INJECTION_API) && CMK_BALANCED_INJECTION_API != 0
884  if(CkMyRank() == 0){
885  // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
886  ck_set_GNI_BIConfig(balancedInjectionLevel);
887  // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
888  }
889 #endif
890 
891  break;
892 
893  case 15:
894  {
895  //For debugging
896  /*if(!CkMyPe()){
897  FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
898  dumpbench(dumpFile);
899  NAMD_die("Normal execution\n");
900  }*/
901  }
902  #ifdef MEM_OPT_VERSION
903  //free space in the Molecule object that are not used anymore
904  ioMgr->freeMolSpace();
905  #endif
906  gotoRun = true;
907  break;
908 
909  default:
910  NAMD_bug("Startup Phase has a bug - check case statement");
911  break;
912 
913  }
914 
915  startupPhase++;
916  if (!CkMyPe()) {
917  if (!gotoRun) {
918  CkStartQD(CkCallback(CkIndex_Node::startup(), thisgroup));
919  } else {
921  }
922  }
923 
924  NAMD_EVENT_STOP(1, NamdProfileEvent::NAMD_STARTUP);
925 }
926 
927 #ifdef OPENATOM_VERSION
928 void Node::doneMoaStart()
929 {
930 #ifdef OPENATOM_VERSION_DEBUG
931  CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
932 #endif //OPENATOM_VERSION_DEBUG
933 }
934 #endif //OPENATOM_VERSION
935 
936 void Node::namdOneCommInit()
937 {
938  if (CkpvAccess(comm) == NULL) {
939  CkpvAccess(comm) = new Communicate();
940 #ifdef DPMTA
941  pvmc_init();
942 #endif
943  }
944 }
945 
946 // Namd 1.X style Send/Recv of simulation information
947 
948 void Node::namdOneRecv() {
949  if ( CmiMyRank() ) return;
950 
951  MIStream *conv_msg;
952 
953  // Receive molecule and simulation parameter information
955  //****** BEGIN CHARMM/XPLOR type changes
957  //****** END CHARMM/XPLOR type changes
959 
960  DebugM(4, "Getting SimParameters\n");
961  conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
963 
964  DebugM(4, "Getting Parameters\n");
965  conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
966  parameters->receive_Parameters(conv_msg);
967 
968  DebugM(4, "Getting Molecule\n");
969  conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
970  // Modified by JLai -- 10.21.11
971  molecule->receive_Molecule(conv_msg);
973  iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
974  conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
975  molecule->receive_GoMolecule(conv_msg);
976  }
977  // End of modification
978  DebugM(4, "Done Receiving\n");
979 }
980 
981 void Node::namdOneSend() {
985 
986  MOStream *conv_msg;
987  // I'm Pe(0) so I send what I know
988  DebugM(4, "Sending SimParameters\n");
989  conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
991 
992  DebugM(4, "Sending Parameters\n");
993  conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
994  parameters->send_Parameters(conv_msg);
995 
996  DebugM(4, "Sending Molecule\n");
997  int bufSize = BUFSIZE;
998  if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
999  conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
1000  // Modified by JLai -- 10.21.11
1001  molecule->send_Molecule(conv_msg);
1002 
1003  if(simParameters->goForcesOn) {
1004  iout << iINFO << "Master Node sending GoMolecule Information" << "\n" << endi;
1005  conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
1006  molecule->send_GoMolecule(conv_msg);
1007  } // End of modification
1008 }
1009 
1010 
1011 void Node::reloadStructure(const char *fname, const char *pdbname) {
1012  delete molecule;
1013  molecule = state->molecule = 0;
1014  delete pdb;
1015  pdb = state->pdb = 0;
1016  state->loadStructure(fname,pdbname,1);
1017  this->molecule = state->molecule;
1018  this->pdb = state->pdb;
1019  CProxy_Node nodeProxy(thisgroup);
1020  nodeProxy.resendMolecule();
1021 }
1022 
1023 
1025  if ( CmiMyRank() ) {
1026  return;
1027  }
1028  if ( CmiMyPe() == 0 ) {
1029  int bufSize = BUFSIZE;
1030  MOStream *conv_msg;
1031  conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, bufSize);
1032  parameters->send_Parameters(conv_msg);
1033  if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
1034  conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
1035  molecule->send_Molecule(conv_msg);
1036  } else {
1037  MIStream *conv_msg;
1038  delete parameters;
1039  parameters = new Parameters;
1040  conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
1041  parameters->receive_Parameters(conv_msg);
1042  delete molecule;
1044  conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
1045  molecule->receive_Molecule(conv_msg);
1046  }
1051  CProxy_Node nodeProxy(thisgroup);
1052  for ( int i=0; i<CmiMyNodeSize(); ++i ) {
1053  nodeProxy[CmiMyPe()+i].resendMolecule2();
1054  }
1055 }
1056 
1061 }
1062 
1063 
1064 // Initial thread setup
1065 
1066 void Node::threadInit() {
1067  // Thread initialization
1068  if (CthImplemented()) {
1069  CthSetStrategyDefault(CthSelf());
1070  } else {
1071  NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
1072  }
1073 }
1074 
1075 //
1076 void Node::buildSequencers() {
1079 
1080  // Controller object is only on Pe(0)
1081  if ( ! CkMyPe() ) {
1082  Controller *controller = new Controller(state);
1083  state->useController(controller);
1084 #ifdef NODEGROUP_FORCE_REGISTER
1085  CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
1086  PatchData *pdata = cpdata.ckLocalBranch();
1087  // OK so I need to wait out here if I want to invoke c_out cm
1088  pdata->c_out = controller;
1089 #endif
1090  }
1091 
1092  CmiNodeBarrier();
1093 
1094  // Assign Sequencer to all HomePatch(es)
1095  for (ai=ai.begin(); ai != ai.end(); ai++) {
1096  HomePatch *patch = (*ai).patch;
1097  Sequencer *sequencer = new Sequencer(patch);
1098  patch->useSequencer(sequencer);
1099  }
1100 }
1101 
1102 
1103 
1104 //-----------------------------------------------------------------------
1105 // Node run() - broadcast to all nodes
1106 //-----------------------------------------------------------------------
1108  (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
1109 }
1110 
1111 
1112 //-----------------------------------------------------------------------
1113 // run(void) runs the specified simulation for the specified number of
1114 // steps, overriding the contents of the configuration file
1115 //-----------------------------------------------------------------------
1117 {
1118 // NAMD_EVENT_START(1, NamdProfileEvent::NODE_RUN);
1119 
1120  // Start Controller (aka scalar Sequencer) on Pe(0)
1121 // printf("\n\n I am in Node.C in run method about to call state->runController\n\n");
1122  if ( ! CkMyPe() ) {
1123  state->runController();
1124  }
1125 
1126  DebugM(4, "Starting Sequencers\n");
1127  // Run Sequencer on each HomePatch - i.e. start simulation
1130  for (ai=ai.begin(); ai != ai.end(); ai++) {
1131  HomePatch *patch = (*ai).patch;
1132 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
1133  patch->runSequencer();
1134  }
1135 
1136  if (!CkMyPe()) {
1137  double newTime = CmiWallTimer();
1138  iout << iINFO << "Startup phase " << startupPhase-1 << " took "
1139  << newTime - startupTime << " s, "
1140  << memusage_MB() << " MB of memory in use\n";
1141  iout << iINFO << "Finished startup at " << newTime << " s, "
1142  << memusage_MB() << " MB of memory in use\n\n" << endi;
1143  fflush(stdout);
1144  }
1145 
1146 // NAMD_EVENT_STOP(1, NamdProfileEvent::NODE_RUN);
1147 }
1148 
1149 
1150 //-----------------------------------------------------------------------
1151 // Node scriptBarrier() - twiddle parameters with simulation halted
1152 //-----------------------------------------------------------------------
1153 
1155  CkStartQD(CkIndex_Node::scriptBarrier(), &thishandle);
1156 }
1157 
1159  //script->awaken();
1160 }
1161 
1163  simParameters->scriptSet(msg->param,msg->value);
1164  delete msg;
1165 }
1166 
1167 void Node::reloadCharges(const char *filename) {
1168  FILE *file = fopen(filename,"r");
1169  if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
1170 
1171  int n = molecule->numAtoms;
1172  float *charge = new float[n];
1173 
1174  for ( int i = 0; i < n; ++i ) {
1175  if ( ! fscanf(file,"%f",&charge[i]) )
1176  NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
1177  }
1178 
1179  fclose(file);
1180  CProxy_Node(thisgroup).reloadCharges(charge,n);
1181 #ifdef NODEGROUP_FORCE_REGISTER
1182  if(CkMyPe()==0)
1183  {
1184  CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
1185  cpdata.setDeviceKernelUpdateCounter();
1186  }
1187 #endif
1188  delete [] charge;
1189 }
1190 
1191 void Node::reloadCharges(float charge[], int n) {
1192  molecule->reloadCharges(charge,n);
1193 }
1194 
1195 
1196 // BEGIN gf
1197 void Node::reloadGridforceGrid(const char * key) {
1198  DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
1199 
1200  int gridnum;
1201  MGridforceParams *mgridParams;
1202  if (key == NULL) {
1205  } else {
1206  gridnum = simParameters->mgridforcelist.index_for_key(key);
1207  mgridParams = simParameters->mgridforcelist.find_key(key);
1208  }
1209 
1210  if (gridnum < 0 || mgridParams == NULL) {
1211  NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
1212  }
1213 
1214  GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
1215  if (grid == NULL) {
1216  NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
1217  }
1218  grid->reinitialize(simParameters, mgridParams);
1219 
1220  CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
1221 #ifdef NODEGROUP_FORCE_REGISTER
1222  if(CkMyPe()==0)
1223  {
1224  CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
1225  cpdata.setDeviceKernelUpdateCounter();
1226  }
1227 #endif
1228  DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
1229 }
1230 
1231 void Node::updateGridScale(const char* key, Vector scale) {
1232  DebugM(4, "updateGridScale(char*, Vector) called on node " << CkMyPe() << "\n" << endi);
1233 
1234  int gridnum;
1235  MGridforceParams* mgridParams;
1236  if (key == NULL) {
1239  } else {
1240  gridnum = simParameters->mgridforcelist.index_for_key(key);
1241  mgridParams = simParameters->mgridforcelist.find_key(key);
1242  }
1243 
1244  if (gridnum < 0 || mgridParams == NULL) {
1245  NAMD_die("Node::updateGridScale(char*, Vector): Could not find grid.");
1246  }
1247 
1248  GridforceGrid* grid = molecule->get_gridfrc_grid(gridnum);
1249  if (grid == NULL) {
1250  NAMD_bug("Node::updateGridScale(char*, Vector): grid not found");
1251  }
1252  CProxy_Node(thisgroup).updateGridScale(gridnum, scale.x, scale.y, scale.z);
1253 
1254  DebugM(4, "updateGridScale(char*, Vector) finished\n" << endi);
1255 }
1256 void Node::updateGridScale(int gridnum, float sx, float sy, float sz) {
1257  if (CmiMyRank()) return;
1258  DebugM(4, "updateGridScale(char*, int, float, float, float) called on node " << CkMyPe() << "\n" << endi);
1259 
1260  GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
1261  if (grid == NULL) {
1262  NAMD_bug("Node::updateGridScale(char*, int, float, float, float):grid not found");
1263  }
1264 
1265  Vector scale(sx,sy,sz);
1267  grid->set_scale( scale );
1268 #ifdef NODEGROUP_FORCE_REGISTER
1269  if(CkMyPe()==0)
1270  {
1271  CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
1272  cpdata.setDeviceKernelUpdateCounter();
1273  }
1274 #endif
1275  DebugM(4, "updateGridScale(char*, int, float, float, float) finished\n" << endi);
1276 }
1277 
1278 void Node::reloadGridforceGrid(int gridnum) {
1279  if (CmiMyRank()) return;
1280  DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
1281 
1282  GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
1283  if (grid == NULL) {
1284  NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
1285  }
1286 
1287  if (CkMyPe()) {
1288  // not node 0 -> receive grid
1289  DebugM(4, "Receiving grid\n");
1290 
1291  delete grid;
1292 
1293  MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
1294  grid = GridforceGrid::unpack_grid(gridnum, msg);
1295  molecule->set_gridfrc_grid(gridnum, grid);
1296  delete msg;
1297  } else {
1298  // node 0 -> send grid
1299  DebugM(4, "Sending grid\n");
1300 
1301  MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
1302  GridforceGrid::pack_grid(grid, msg);
1303  msg->end();
1304  delete msg;
1305  }
1306 #ifdef NODEGROUP_FORCE_REGISTER
1307  if(CkMyPe()==0)
1308  {
1309  CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
1310  cpdata.setDeviceKernelUpdateCounter();
1311  }
1312 #endif
1313  DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
1314 }
1315 // END gf
1316 
1317 
1318 // initiating replica
1319 void Node::sendCheckpointReq(int remote, const char *key, int task, Lattice &lat, ControllerState &cs) {
1320  CheckpointMsg *msg = new (1+strlen(key),0) CheckpointMsg;
1321  msg->replica = CmiMyPartition();
1322  msg->task = task;
1323  msg->checkpoint.lattice = lat;
1324  msg->checkpoint.state = cs;
1325  strcpy(msg->key,key);
1326  envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
1327  CmiSetHandler(env,recvCheckpointCReq_index);
1328 #if CMK_HAS_PARTITION
1329  CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
1330 #else
1331  CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
1332 #endif
1333 }
1334 
1335 // responding replica
1336 extern "C" {
1337  void recvCheckpointCReq_handler(envelope *env) {
1338  Node::Object()->recvCheckpointReq(CheckpointMsg::unpack(EnvToUsr(env)));
1339  }
1340 }
1341 
1342 // responding replica
1344  state->controller->recvCheckpointReq(msg->key,msg->task,msg->checkpoint);
1345 
1346  int remote = msg->replica;
1347  msg->replica = CmiMyPartition();
1348  envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
1349  CmiSetHandler(env,recvCheckpointCAck_index);
1350 #if CMK_HAS_PARTITION
1351  CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
1352 #else
1353  CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
1354 #endif
1355 }
1356 
1357 // initiating replica
1358 extern "C" {
1359  void recvCheckpointCAck_handler(envelope *env) {
1360  Node::Object()->recvCheckpointAck(CheckpointMsg::unpack(EnvToUsr(env)));
1361  }
1362 }
1363 
1364 // initiating replica
1366  state->controller->recvCheckpointAck(msg->checkpoint);
1367  delete msg;
1368 }
1369 
1370 
1372  //CmiPrintf("sendEnableExitScheduler\n");
1373  CProxy_Node nodeProxy(thisgroup);
1374  nodeProxy[0].recvEnableExitScheduler();
1375 }
1376 
1378  //CmiPrintf("recvEnableExitScheduler\n");
1380 }
1381 
1383  if ( CkMyPe() ) {
1385  } else {
1386  CkStartQD(CkIndex_Node::exitScheduler(), &thishandle);
1387  }
1388 }
1389 
1391  //CmiPrintf("exitScheduler %d\n",CkMyPe());
1392  CsdExitScheduler();
1393 }
1394 
1396  CProxy_Node nodeProxy(thisgroup);
1397  nodeProxy[0].recvEnableEarlyExit();
1398 }
1399 
1401  enableEarlyExit();
1402 }
1403 
1405  if ( CkMyPe() ) {
1407  } else {
1408  CkStartQD(CkIndex_Node::earlyExit(),&thishandle);
1409  }
1410 }
1411 
1412 void Node::earlyExit(void) {
1413  NAMD_die("Exiting prematurely; see error messages above.");
1414 }
1415 
1416 
1417 //------------------------------------------------------------------------
1418 // Some odd utilities
1419 //------------------------------------------------------------------------
1421 {
1422  this->molecule = state->molecule;
1423  this->parameters = state->parameters;
1424  this->simParameters = state->simParameters;
1425  this->configList = state->configList;
1426  this->pdb = state->pdb;
1427  this->state = state;
1428 }
1429 
1430 // entry methods for BG/P HPM (performance counters) library
1432 #if USE_HPM
1433  HPM_Start("500 steps", localRankOnNode);
1434 #endif
1435 }
1436 
1438 #if USE_HPM
1439  HPM_Stop("500 steps", localRankOnNode);
1440  HPM_Print(CkMyPe(), localRankOnNode);
1441 #endif
1442 }
1443 
1444 void Node::traceBarrier(int turnOnTrace, int step){
1445  curTimeStep = step;
1446  if(turnOnTrace) traceBegin();
1447  else traceEnd();
1448 
1449  if(turnOnTrace) CmiTurnOnStats();
1450  else CmiTurnOffStats();
1451 
1452  //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());
1453  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
1454  CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
1455  contribute(0, NULL, CkReduction::sum_int, cb);
1456 
1457 }
1458 
1459 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
1460  CmiAssert(CmiMyPe()==0);
1461  delete msg;
1463 }
1464 
1465 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
1466 #ifdef MEASURE_NAMD_WITH_PAPI
1467  curMFlopStep = step;
1468  double results[NUM_PAPI_EVENTS+1];
1469 
1470  if(turnOnMeasure){
1471  CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]=CmiWallTimer();
1472 
1473  long long counters[NUM_PAPI_EVENTS+1];
1474  int ret=PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
1475  if(ret==PAPI_OK)
1476  {
1477  // CkPrintf("traceBarrier start counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
1478  }
1479  else
1480  {
1481  CkPrintf("error PAPI_start_counters (%d) at step %d called on proc %d\n",ret , step, CkMyPe());
1482  }
1483  if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)!=PAPI_OK)
1484  {
1485  CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
1486  };
1487  }else{
1488  long long counters[NUM_PAPI_EVENTS+1];
1489  for(int i=0;i<NUM_PAPI_EVENTS;i++) counters[i]=0LL;
1490  if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)==PAPI_OK)
1491  {
1492 #if !MEASURE_PAPI_SPP
1493  results[0] = (double)counters[0]/1e6;
1494  results[1] = (double)counters[1]/1e6;
1495 #else
1496  for(int i=0;i<NUM_PAPI_EVENTS;i++) results[i] = counters[i]/1e6;
1497 #endif
1498  // for(int i=0;i<NUM_PAPI_EVENTS;i++) CkPrintf("[%d] counter %d is %ld\n",CkMyPe(),i,counters[i]);
1499  }
1500  else
1501  {
1502  // CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
1503  }
1504  // CkPrintf("traceBarrier stop counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
1505 
1506  PAPI_stop_counters(counters, NUM_PAPI_EVENTS);
1507  }
1508  if(CkMyPe()==0)
1509  // CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
1510  results[NUM_PAPI_EVENTS]=CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]; //starttime
1511  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
1512  CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
1513  contribute(sizeof(double)*(NUM_PAPI_EVENTS+1), &results, CkReduction::sum_double, cb);
1514 #endif
1515 }
1516 
1517 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
1518 #ifdef MEASURE_NAMD_WITH_PAPI
1519 
1520  if(simParameters->papiMeasureStartStep != curMFlopStep) {
1521  double *results = (double *)msg->getData();
1522  double endtime=CmiWallTimer();
1523  int bstep = simParameters->papiMeasureStartStep;
1524  int estep = bstep + simParameters->numPapiMeasureSteps;
1525 #if MEASURE_PAPI_SPP
1526  CkPrintf("SPP INFO: PAPI_FP_OPS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[0]);
1527  CkPrintf("SPP INFO: PAPI_TOT_INS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[1]);
1528  CkPrintf("SPP INFO: perf::PERF_COUNT_HW_CACHE_LL:MISS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[2]);
1529  CkPrintf("SPP INFO: DATA_PREFETCHER:ALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[3]);
1530  CkPrintf("SPP INFO: PAPI_L1_DCA timestep %d to %d is %lf(1e6)\n", bstep,estep,results[4]);
1531  CkPrintf("SPP INFO: PAPI_TOT_CYC timestep %d to % is %lf(1e6)\n", bstep,estep,results[5]);
1532  // CkPrintf("SPP INFO: INSTRUCTION_FETCH_STALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[6]);
1533  // CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]/CkNumPes());
1534  CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]);
1535  CkPrintf("SPP INFO: endtime %lf avgtime %lf tottime %lf\n", endtime,results[NUM_PAPI_EVENTS]/CkNumPes(),results[NUM_PAPI_EVENTS] );
1536 #else
1537  if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
1538  double totalFPIns = results[0];
1539  if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
1540  CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n",
1541  bstep, estep, totalFPIns/CkNumPes());
1542  }else{
1543  char nameBuf[PAPI_MAX_STR_LEN];
1544  CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ",
1545  bstep, estep);
1546  for(int i=0; i<NUM_PAPI_EVENTS; i++) {
1547  PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
1548  CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
1549  }
1550  CkPrintf("per processor\n");
1551  }
1552 #endif
1553  }
1554  delete msg;
1555  state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
1556 #endif
1557 }
1558 
1559 extern char *gNAMDBinaryName;
1560 void Node::outputPatchComputeMaps(const char *filename, int tag){
1562 
1563  int numpes = CkNumPes();
1564  int nodesize = CkMyNodeSize();
1566  numpes = simParameters->simulatedPEs;
1567  nodesize = simParameters->simulatedNodeSize;
1568  }
1569 
1570  char fname[128];
1571  sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
1572 
1573  FILE *fp = fopen(fname, "w");
1574  if(fp == NULL) {
1575  NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
1576  return;
1577  }
1578  PatchMap *pMap = PatchMap::Object();
1579  ComputeMap *cMap = ComputeMap::Object();
1580  int numPatches = pMap->numPatches();
1581  int numComputes = cMap->numComputes();
1582  fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes,
1583  pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
1584  //output PatchMap info
1585  for(int i=0; i<numPatches; i++) {
1586  #ifdef MEM_OPT_VERSION
1587  fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
1588  #else
1589  fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
1590  #endif
1591  }
1592 
1593  //output ComputeMap info
1594  for(int i=0; i<numComputes; i++) {
1595  fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));
1596  }
1597 }
1598 
1600 #ifndef NODEGROUP_FORCE_REGISTER
1601  return script;
1602 #else
1603  if(script == NULL){
1604  CProxy_PatchData cpdata(CkpvAccess(BOCclass_group).patchData);
1605  PatchData* patchData = cpdata.ckLocalBranch();
1606  return patchData->script;
1607  }else{
1608  return script;
1609  }
1610 #endif
1611 }
1612 
1613 
1614 //======================================================================
1615 // Private functions
1616 
1617 #include "Node.def.h"
1618 
static Node * Object()
Definition: Node.h:86
#define GRIDFORCEGRIDTAG
Definition: common.h:183
void allocateMap(int nAtomIDs)
Definition: AtomMap.C:161
#define NAMD_EVENT_STOP(eon, id)
std::ostream & iINFO(std::ostream &s)
Definition: InfoStream.C:81
bool specialTracing
Definition: Node.h:166
Bool simulateInitialMapping
void recvCheckpointReq(const char *key, int task, checkpoint &cp)
Definition: Controller.C:5161
int getNumAtoms() const
Definition: Patch.h:105
void recvCheckpointCReq_handler(envelope *)
Definition: Node.C:1337
void setPatchMapArrived(bool s)
Definition: WorkDistrib.h:107
void runSequencer(void)
Definition: HomePatch.C:305
void createProxies(void)
Definition: ProxyMgr.C:416
GridforceGrid * get_gridfrc_grid(int gridnum) const
Definition: Molecule.h:1363
void end(void)
Definition: MStream.C:176
ControllerState state
Definition: Controller.h:388
void setRecvSpanning()
Definition: ProxyMgr.C:370
void receive_SimParameters(MIStream *)
int proxyRecvSpanning
Definition: ProxyMgr.C:45
BOCgroup group
Definition: Node.h:68
int numComputes(void)
Definition: ComputeMap.h:101
int getRecvSpanning()
Definition: ProxyMgr.C:375
int curTimeStep
Definition: Node.h:152
void send_GoMolecule(MOStream *)
Definition: GoMolecule.C:1636
void mallocTest(int)
Definition: Node.C:400
void startHPM()
Definition: Node.C:1431
static ProxyMgr * Object()
Definition: ProxyMgr.h:394
NAMD_HOST_DEVICE int c_p() const
Definition: Lattice.h:291
void exitScheduler(void)
Definition: Node.C:1390
IMDOutput * imd
Definition: Node.h:186
void saveMolDataPointers(NamdState *)
Definition: Node.C:1420
void receive_GoMolecule(MIStream *)
Definition: GoMolecule.C:1745
LdbCoordinator * ldbCoordinator
Definition: Node.h:205
static PatchMap * Object()
Definition: PatchMap.h:27
void sendEnableEarlyExit(void)
Definition: Node.C:1395
void send_Molecule(MOStream *)
Definition: Molecule.C:5589
static void exit(int status=0)
Definition: BackEnd.C:277
Definition: Vector.h:72
static AtomMap * Instance()
Definition: AtomMap.C:125
Output * output
Definition: Node.h:185
SimParameters * simParameters
Definition: Node.h:181
int task
Definition: Node.C:94
void setSendSpanning()
Definition: ProxyMgr.C:361
static void pack_grid(GridforceGrid *grid, MOStream *msg)
Definition: GridForceGrid.C:50
#define DebugM(x, y)
Definition: Debug.h:75
void createLoadBalancer()
Controller * c_out
Definition: PatchData.h:141
double startupTime
Definition: Node.C:294
HomePatchList * homePatchList()
Definition: PatchMap.C:438
std::ostream & endi(std::ostream &s)
Definition: InfoStream.C:54
#define ALLBUTME
Definition: Communicate.h:14
BigReal z
Definition: Vector.h:74
void enableScriptBarrier()
Definition: Node.C:1154
int getSendSpanning()
Definition: ProxyMgr.C:366
void scriptSet(const char *, const char *)
char value[MAX_SCRIPT_PARAM_SIZE]
Definition: Node.h:75
void sendEnableExitScheduler(void)
Definition: Node.C:1371
void recvCheckpointReq(CheckpointMsg *)
Definition: Node.C:1343
static void messageStartUp()
Definition: Node.C:427
ResizeArrayIter< T > begin(void) const
void stopHPM()
Definition: Node.C:1437
void reloadCharges(float charge[], int n)
#define iout
Definition: InfoStream.h:51
Patch * patch(PatchID pid)
Definition: PatchMap.h:244
int curMFlopStep
Definition: Node.h:157
#define STATICPARAMSTAG
Definition: common.h:178
Molecule * node_molecule
Definition: Node.C:433
void outputPatchComputeMaps(const char *filename, int tag)
Definition: Node.C:1560
int loadStructure(const char *, const char *, int)
Definition: NamdState.C:163
void createComputes(ComputeMap *map)
Definition: ComputeMgr.C:973
Molecule stores the structural information for the system.
Definition: Molecule.h:175
NAMD_HOST_DEVICE int b_p() const
Definition: Lattice.h:290
int replica
Definition: Node.C:95
void split(int iStream, int numStreams)
Definition: Random.h:77
void recvCheckpointAck(CheckpointMsg *)
Definition: Node.C:1365
void patchMapInit(void)
Definition: WorkDistrib.C:1226
void openSync()
Definition: Sync.C:63
ComputeMap * computeMap
Definition: Node.h:204
int gridsize_c(void) const
Definition: PatchMap.h:66
void runController(void)
Definition: NamdState.C:83
double memusage_MB()
Definition: memusage.h:13
virtual void reinitialize(SimParameters *simParams, MGridforceParams *mgridParams)=0
void sendComputeMap(void)
Definition: WorkDistrib.C:1199
Definition: Output.h:35
int gridsize_a(void) const
Definition: PatchMap.h:64
void resumeAfterPapiMeasureBarrier(CkReductionMsg *msg)
Definition: Node.C:1517
void initialize(PatchMap *pmap, ComputeMap *cmap, int reinit=0)
void scriptParam(ScriptParamMsg *)
Definition: Node.C:1162
Definition: Random.h:37
int index_for_key(const char *key)
void run()
Definition: Node.C:1116
int numPatches(void) const
Definition: PatchMap.h:59
#define NAMD_EVENT_START(eon, id)
void enableExitScheduler(void)
Definition: Node.C:1382
void buildProxySpanningTree()
Definition: ProxyMgr.C:558
void createHomePatches(void)
Definition: WorkDistrib.C:977
void NAMD_bug(const char *err_msg)
Definition: common.C:195
char * gNAMDBinaryName
Definition: BackEnd.C:240
ComputeType type(ComputeID cid)
Definition: ComputeMap.C:118
ScaledPosition smax
Definition: ComputeMsm.h:21
Controller::checkpoint checkpoint
Definition: Node.C:96
NamdState * state
Definition: Node.h:184
#define MGRIDFORCEPARAMS_DEFAULTKEY
MGridforceParams * find_key(const char *key)
void traceBarrier(int turnOnTrace, int step)
Definition: Node.C:1444
ScriptTcl * script
Definition: PatchData.h:154
void updateGridScale(const char *key, Vector scale)
Definition: Node.C:1231
static void pme_select()
static void messageRun()
Definition: Node.C:1107
void scriptBarrier(void)
Definition: Node.C:1158
static Sync * Object()
Definition: Sync.h:52
void recvCheckpointAck(checkpoint &cp)
Definition: Controller.C:5191
void reloadStructure(const char *, const char *)
Definition: Node.C:1011
BigReal x
Definition: Vector.h:74
AtomMap * atomMap
Definition: Node.h:202
void recvEnableExitScheduler(void)
Definition: Node.C:1377
void get_extremes(ScaledPosition &xmin, ScaledPosition &xmax) const
Definition: PDB.h:104
NAMD_HOST_DEVICE int a_p() const
Definition: Lattice.h:289
int numAtoms
Definition: Molecule.h:585
void NAMD_die(const char *err_msg)
Definition: common.C:147
PDB * pdb
Definition: Node.h:183
static LdbCoordinator * Object()
ConfigList * configList
Definition: Node.h:182
static AtomMap * Object()
Definition: AtomMap.h:37
#define BUFSIZE
Definition: Communicate.h:15
MGridforceParamsList mgridforcelist
static void nonbonded_select()
void recvEnableEarlyExit(void)
Definition: Node.C:1400
void sendPatchMap(void)
Definition: WorkDistrib.C:1099
void send_Parameters(MOStream *)
Definition: Parameters.C:6560
int isRecvSpanningTreeOn()
Parameters * parameters
Definition: Node.h:180
static ComputeMap * Instance()
Definition: ComputeMap.C:24
void resumeAfterTraceBarrier(CkReductionMsg *msg)
Definition: Node.C:1459
unsigned int randomSeed
CkpvDeclare(AtomMap *, AtomMap_instance)
WorkDistrib * workDistrib
Definition: Node.h:169
~Node(void)
Definition: Node.C:363
Parameters * node_parameters
Definition: Node.C:432
static GridforceGrid * unpack_grid(int gridnum, MIStream *msg)
Definition: GridForceGrid.C:60
#define SIMPARAMSTAG
Definition: common.h:177
SimParameters * node_simParameters
Definition: Node.C:431
Random * rand
Definition: Node.h:175
void mallocTestQd(void)
Definition: Node.C:412
void mapComputes(void)
Definition: WorkDistrib.C:2395
void recvCheckpointCAck_handler(envelope *)
Definition: Node.C:1359
void registerUserEventsForAllComputeObjs(void)
Definition: ComputeMgr.C:794
static ComputeMap * Object()
Definition: ComputeMap.h:89
PatchMap * patchMap
Definition: Node.h:203
ScaledPosition smin
Definition: ComputeMsm.h:21
void useController(Controller *controllerPtr)
Definition: NamdState.C:78
void papiMeasureBarrier(int turnOnMeasure, int step)
Definition: Node.C:1465
BigReal y
Definition: Vector.h:74
void resendMolecule2()
Definition: Node.C:1057
ScriptTcl * getScript()
Definition: Node.C:1599
void resendMolecule()
Definition: Node.C:1024
void distributeHomePatches(void)
Definition: WorkDistrib.C:1051
void setProxyTreeBranchFactor(int dim)
Definition: ProxyMgr.C:379
colvarmodule * colvars
Definition: Node.h:187
int eventEndOfTimeStep
Definition: Node.C:293
void startup()
Definition: Node.C:437
int node(ComputeID cid)
Definition: ComputeMap.h:106
int gridsize_b(void) const
Definition: PatchMap.h:65
int set_gridfrc_grid(int gridnum, GridforceGrid *grid)
Definition: Molecule.h:1372
__thread DeviceCUDA * deviceCUDA
Definition: DeviceCUDA.C:23
int pid(ComputeID cid, int i)
Definition: ComputeMap.C:107
#define MOLECULETAG
Definition: common.h:179
int isSendSpanningTreeOn()
Node(GroupInitMsg *msg)
Definition: Node.C:298
void resumeAfterTraceBarrier(int)
Definition: Controller.C:5231
void sendCheckpointReq(int remote, const char *key, int task, Lattice &lat, ControllerState &cs)
Definition: Node.C:1319
ComputeMgr * computeMgr
Definition: Node.h:172
int node(int pid) const
Definition: PatchMap.h:114
int mallocTest_size
Definition: Node.h:130
void enableEarlyExit(void)
Definition: Node.C:1404
void receive_Molecule(MIStream *)
Definition: Molecule.C:5955
void earlyExit(void)
Definition: Node.C:1412
MGridforceParams * at_index(int idx)
ResizeArrayIter< T > end(void) const
static PatchMap * Instance()
Definition: PatchMap.C:32
Molecule * molecule
Definition: Node.h:179
void useSequencer(Sequencer *sequencerPtr)
Definition: HomePatch.C:301
char param[MAX_SCRIPT_PARAM_SIZE]
Definition: Node.h:74
void receive_Parameters(MIStream *)
Definition: Parameters.C:6936
virtual void set_scale(Vector s)=0
void reloadCharges(const char *filename)
Definition: Node.C:1167
void send_SimParameters(MOStream *)
void reloadGridforceGrid(const char *key)
Definition: Node.C:1197
void assignNodeToPatch(void)
Definition: WorkDistrib.C:1444
char * key
Definition: Node.C:97
int proxySendSpanning
Definition: ProxyMgr.C:44