Node.C

Go to the documentation of this file.
00001 
00007 /*
00008    Toplevel routines for initializing a Node for a simulation
00009    one Node per Pe (processor element).
00010 */
00011 
00012 #if !defined(WIN32) || defined(__CYGWIN__)
00013 #include <unistd.h>
00014 #endif
00015 #include "InfoStream.h"
00016 #include "Node.decl.h"
00017 #include "Node.h"
00018 #ifdef DPMTA
00019 #include <pvm3.h>
00020 #endif
00021 
00022 #include "ProcessorPrivate.h"
00023 
00024 #define MIN_DEBUG_LEVEL 3
00025 //#define DEBUGM
00026 #include "Debug.h"
00027 
00028 #include <stdio.h>
00029 #include <converse.h>
00030 #include "memusage.h"
00031 #include "IMDOutput.h"
00032 #include "Lattice.h"
00033 #include "ComputeMsmMsa.h"  // needed for MsmMsaData definition
00034 #include "ComputeMsm.h"     // needed for MsmInitMsg definition
00035 #include "main.decl.h"
00036 #include "main.h"
00037 #include "WorkDistrib.h"
00038 #include "PatchMgr.h"
00039 #include "Patch.h"
00040 #include "Compute.h"
00041 #include "ComputeMap.h"
00042 #include "ComputeMgr.h"
00043 #include "Molecule.h"
00044 #include "HomePatchList.h"
00045 #include "AtomMap.h"
00046 #include "Sequencer.h"
00047 #include "Controller.h"
00048 #include "NamdState.h"
00049 #include "Output.h"
00050 #include "ProxyMgr.h"
00051 #include "PatchMap.h"
00052 #include "PatchMap.inl"
00053 #include "Parameters.h"
00054 #include "SimParameters.h"
00055 #include "Communicate.h"
00056 #include "LdbCoordinator.h"
00057 #include "ScriptTcl.h"
00058 #include "ComputeMgr.decl.h"
00059 #include "ComputePmeMgr.decl.h"
00060 // #ifdef NAMD_CUDA
00061 #include "ComputeCUDAMgr.decl.h"
00062 #include "ComputeCUDAMgr.h"
00063 #include "ComputePmeCUDAMgr.decl.h"
00064 #include "ComputePmeCUDAMgr.h"
00065 // #endif
00066 #include "ComputeGridForceMgr.decl.h"
00067 #include "OptPmeMgr.decl.h"
00068 #include "Sync.h"
00069 #include "BackEnd.h"
00070 #include "PDB.h"
00071 #include "packmsg.h"
00072 #include "CollectionMgr.decl.h"
00073 #include "ParallelIOMgr.decl.h"
00074 #include "Vector.h"
00075 // BEGIN LA
00076 #include "Random.h"
00077 // END LA
00078 
00079 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00080 extern "C" void CApplicationInit();
00081 #endif
00082 
00083 #include "DumpBench.h"
00084 
00085 class CheckpointMsg : public CMessage_CheckpointMsg {
00086 public:
00087   int task;
00088   int replica;
00089   Controller::checkpoint checkpoint;
00090   char *key;
00091 };
00092 
00093 extern "C" {
00094   void recvCheckpointCReq_handler(envelope*);
00095   void recvCheckpointCAck_handler(envelope*);
00096 }
00097 
00098 #ifdef CMK_BALANCED_INJECTION_API
00099 #include "ckBIconfig.h"
00100 #endif
00101 
00102 #include "CollectionMgr.h"
00103 #include "CollectionMaster.h"
00104 #include "CollectionMgr.decl.h"
00105 #include "CollectionMaster.decl.h"
00106 
00107 #if USE_HPM
00108 extern "C" void HPM_Init(int);
00109 extern "C" void HPM_Start(char *label, int);
00110 extern "C" void HPM_Stop(char *label, int);
00111 extern "C" void HPM_Print(int, int);
00112 #endif
00113 
00114 #if defined(NAMD_MIC)
00115   extern void mic_dumpHostDeviceComputeMap();
00116   extern void mic_initHostDeviceLDB();
00117 #endif
00118 
00119 #ifdef MEASURE_NAMD_WITH_PAPI
00120 #include "papi.h"
00121 #if CMK_SMP
00122 #include <pthread.h>
00123 #endif
00124 #define NUM_PAPI_EVENTS 6
00125 CkpvDeclare(int *, papiEvents);
00126 
00127 #define MEASURE_PAPI_SPP 1
00128 #define MEASURE_PAPI_CACHE 0
00129 #define MEASURE_PAPI_FLOPS 0
00130 
00131 static void namdInitPapiCounters(){
00132         if(CkMyRank()==0){
00133                 //only initialize per OS process (i.e. a charm node)
00134                 int retval = PAPI_library_init(PAPI_VER_CURRENT);
00135                 if(retval != PAPI_VER_CURRENT) {
00136                         if(CkMyPe()==0){
00137                                 NAMD_die("PAPI library is not compatitible!");
00138                         }
00139                 }
00140         #if CMK_SMP
00141                 //now only consider systems that are compatible with POSIX
00142                 if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
00143                         if(CkMyPe()==0){
00144                                 NAMD_die("Multi-thread mode in PAPI could not be initialized!");
00145                         }
00146                 }
00147         #endif
00148         }
00149         CkpvInitialize(int *, papiEvents);
00150         CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS+1];
00151 
00152 #if MEASURE_PAPI_CACHE
00153         if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
00154                 CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
00155         }else{
00156                 if(CkMyPe()==0){
00157                         CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");                     
00158                 }
00159                 //if not default to PAPI_TOT_INS
00160                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00161         }
00162 
00163         if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
00164                 CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
00165         }else{
00166                 //if not default to PAPI_TOT_CYC
00167                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00168         }       
00169 #elif MEASURE_PAPI_FLOPS
00170         if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
00171                 CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
00172         }else{
00173                 if(CkMyPe()==0){
00174                         CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
00175                 }
00176                 //if not default to PAPI_TOT_INS
00177                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00178         }
00179 
00180         if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
00181                 CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
00182         }else{
00183                 //if not default to PAPI_TOT_CYC
00184                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00185         }
00186 #elif MEASURE_PAPI_SPP
00187 /* for SPP we record these
00188 1) PAPI_FP_OPS
00189 2) PAPI_TOT_INS
00190 3) perf::PERF_COUNT_HW_CACHE_LL:MISS
00191 4) DATA_PREFETCHER:ALL
00192 5) PAPI_L1_DCA
00193 6) INSTRUCTION_FETCH_STALL
00194 7) PAPI_TOT_CYC, and 
00195 8) real (wall) time
00196 */
00197         int papiEventSet = PAPI_NULL; 
00198         if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
00199           CmiAbort("PAPI failed to create event set!\n");
00200         }
00201 
00202         if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
00203                 CkpvAccess(papiEvents)[0] = PAPI_FP_OPS;
00204         }else{
00205                 if(CkMyPe()==0){
00206                         CkAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
00207                 }
00208         }
00209         if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
00210                 CkpvAccess(papiEvents)[1] = PAPI_TOT_INS;
00211         }else{
00212                 if(CkMyPe()==0){
00213                         CkAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
00214                 }
00215         }
00216         int EventCode;
00217         int ret;
00218         ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
00219         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00220           CkpvAccess(papiEvents)[2] = EventCode;
00221         }else{
00222                 if(CkMyPe()==0){
00223                         CkAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
00224                 }
00225         }
00226         ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
00227         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00228           CkpvAccess(papiEvents)[3] = EventCode;
00229         }else{
00230                 if(CkMyPe()==0){
00231                         CkAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
00232                 }
00233         }
00234         if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
00235                 CkpvAccess(papiEvents)[4] = PAPI_L1_DCA;
00236         }else{
00237                 if(CkMyPe()==0){
00238                         CkAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
00239                 }
00240         }
00241         /*      ret=PAPI_event_name_to_code("INSTRUCTION_FETCH_STALL",&EventCode);
00242         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00243           CkpvAccess(papiEvents)[5] = EventCode;
00244         }else{
00245                 if(CkMyPe()==0){
00246                         CkAbort("WARNING: INSTRUCTION_FETCH_STALL doesn't exist on this platform!");
00247                 }
00248         }
00249         */
00250         if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
00251                 CkpvAccess(papiEvents)[5] = PAPI_TOT_CYC;
00252         }else{
00253                 if(CkMyPe()==0){
00254                         CkAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
00255                 }
00256         }
00257         for(int i=0;i<NUM_PAPI_EVENTS;i++)
00258           {
00259             int papiRetValue=PAPI_add_events(papiEventSet, &CkpvAccess(papiEvents)[i],1);
00260             if (papiRetValue != PAPI_OK) {
00261               CkPrintf("failure for event %d\n",i);
00262               if (papiRetValue == PAPI_ECNFLCT) {
00263                 CmiAbort("PAPI events conflict! Please re-assign event types!\n");
00264               } else {
00265                 CmiAbort("PAPI failed to add designated events!\n");
00266               }
00267             }
00268             
00269           }
00270 #endif
00271 }
00272 #endif
00273 
00274 #ifdef OPENATOM_VERSION
00275 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
00276 {
00277   CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
00278 }
00279 #endif //OPENATOM_VERSION
00280 
00281 //======================================================================
00282 // Public Functions
00283 
00284 //----------------------------------------------------------------------
00285 
00286 int eventEndOfTimeStep;
00287 double startupTime;
00288 
00289 //----------------------------------------------------------------------
00290 // BOC constructor
00291 Node::Node(GroupInitMsg *msg)
00292 {    
00293   DebugM(4,"Creating Node\n");
00294 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00295   CApplicationInit();
00296 #endif
00297   if (CkpvAccess(Node_instance) == 0) {
00298     CkpvAccess(Node_instance) = this;
00299     eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep", 135);
00300   } else {
00301     NAMD_bug("Node::Node() - another instance of Node exists!");
00302   }
00303 
00304   CkpvAccess(BOCclass_group) = msg->group;
00305   delete msg;
00306 
00307   CkpvAccess(BOCclass_group).node = thisgroup;
00308 
00309   recvCheckpointCReq_index = CmiRegisterHandler((CmiHandler)recvCheckpointCReq_handler);
00310   recvCheckpointCAck_index = CmiRegisterHandler((CmiHandler)recvCheckpointCAck_handler);
00311 
00312   startupPhase = 0;
00313 
00314   molecule = NULL;
00315   parameters = NULL;
00316   simParameters = NULL;
00317   configList = NULL;
00318   pdb = NULL;
00319   state = NULL;
00320   output = NULL;
00321   imd = new IMDOutput;
00322   colvars = 0;
00323 
00324 #if USE_HPM
00325   // assumes that this will be done only on BG/P
00326   TopoManager *tmgr = new TopoManager();
00327   int x, y, z;
00328   tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
00329   delete tmgr;
00330 #endif
00331 
00332   specialTracing = traceAvailable() && (traceIsOn()==0);
00333 
00334   DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
00335   patchMap = PatchMap::Instance();
00336   atomMap = AtomMap::Instance();
00337   if ( CkMyRank() == 0 ) ComputeMap::Instance();
00338 
00339   //Note: Binding BOC vars such as workDistrib has been moved
00340   //to the 1st phase of startup because the in-order message delivery
00341   //is not always guaranteed --Chao Mei
00342 #ifdef CMK_BALANCED_INJECTION_API
00343   if(CkMyRank() == 0){
00344     balancedInjectionLevel=ck_get_GNI_BIConfig();
00345     // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00346     ck_set_GNI_BIConfig(20);
00347     // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00348   }
00349 #endif
00350 
00351 }
00352 
00353 //----------------------------------------------------------------------
00354 // ~Node(void) needs to clean up everything.
00355 
00356 Node::~Node(void)
00357 {
00358   delete output;
00359   delete computeMap;
00360   delete atomMap;
00361   delete patchMap;
00362   delete CkpvAccess(comm);
00363   // BEGIN LA
00364   delete rand;
00365   // END LA
00366 #ifdef MEASURE_NAMD_WITH_PAPI
00367   delete CkpvAccess(papiEvents);
00368 #endif
00369 }
00370 
00371 void Node::bindBocVars(){
00372     DebugM(4,"Binding to BOC's\n");
00373     CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00374     patchMgr = pm.ckLocalBranch();
00375     CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
00376     proxyMgr = prm.ckLocalBranch();
00377     CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
00378     workDistrib = wd.ckLocalBranch();
00379     CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00380     computeMgr = cm.ckLocalBranch();
00381     CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
00382     ldbCoordinator = lc.ckLocalBranch();
00383   #ifdef MEM_OPT_VERSION      
00384     CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
00385     ioMgr = io.ckLocalBranch();
00386   #endif
00387 
00388 }
00389 
00390 //----------------------------------------------------------------------
00391 // Malloc Test Sequence
00392 void Node::mallocTest(int step) {
00393   int MB = 1024*1024;
00394   int size = 100;
00395   char* foo = (char*) malloc(size*MB);
00396   if ( ! foo ) {
00397     char buf[256];
00398     sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
00399     NAMD_die(buf);
00400   }
00401   memset(foo,0,size*MB*sizeof(char));
00402 }
00403 
00404 void Node::mallocTestQd(CkQdMsg *qmsg) {
00405   delete qmsg;
00406   if ( mallocTest_size ) {
00407     CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
00408   } else {
00409     CkPrintf("Starting malloc test on all PEs.\n");
00410   }
00411   fflush(stdout);
00412   ++mallocTest_size;
00413   CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00414   (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
00415 }
00416 
00417 //----------------------------------------------------------------------
00418 // Startup Sequence
00419 
00420 void Node::messageStartUp() {
00421   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00422 }
00423 
00424 void Node::startUp(CkQdMsg *qmsg) {
00425   delete qmsg;
00426   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00427 }
00428 
00429 SimParameters *node_simParameters;
00430 Parameters *node_parameters;
00431 Molecule *node_molecule;
00432 
00433 extern void registerUserEventsForAllComputeObjs(void);
00434 
00435 void Node::startup() {
00436   int gotoRun = false;
00437   double newTime;
00438 
00439   if (!CkMyPe()) {
00440     if (!startupPhase) {
00441       iout << iINFO << "\n";
00442       startupTime = CmiWallTimer();
00443       iout << iINFO << "Entering startup at " << startupTime << " s, ";
00444     } else {
00445       newTime = CmiWallTimer();
00446       iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00447            << newTime - startupTime << " s, ";
00448       startupTime = newTime;
00449     }
00450     iout << memusage_MB() << " MB of memory in use\n" << endi;
00451     fflush(stdout);
00452   }
00453   switch (startupPhase) {
00454 
00455   case 0:
00456     computeMap = ComputeMap::Object();
00457     namdOneCommInit(); // Namd1.X style
00458   break;
00459 
00460   case 1:
00461       bindBocVars();
00462 
00463     // send & receive molecule, simparameters... (Namd1.X style)
00464     if (CkMyPe()) {
00465       namdOneRecv();
00466     } else {
00467       namdOneSend();
00468     }
00469   break;
00470 
00471   case 2:
00472     // fix up one-per-node objects (for SMP version)
00473     simParameters = node_simParameters;
00474     parameters = node_parameters;
00475     molecule = node_molecule;
00476 
00477     SimParameters::nonbonded_select();
00478     if ( simParameters->PMEOn ) SimParameters::pme_select();   
00479  
00480     #if !CMK_SMP || ! USE_CKLOOP
00481     //the CkLoop library should be only used in SMP mode
00482     simParameters->useCkLoop = 0;
00483     #else
00484     if ( CkNumPes() < 2 * CkNumNodes() ) simParameters->useCkLoop = 0;
00485     #endif
00486 
00487 
00488     if ( simParameters->mallocTest ) {
00489       if (!CkMyPe()) {
00490         mallocTest_size = 0;
00491         CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00492       }
00493       return;
00494     }
00495 
00496       
00497         #ifdef MEASURE_NAMD_WITH_PAPI
00498         if(simParameters->papiMeasure) namdInitPapiCounters();  
00499         #endif
00500     
00501     #ifdef MEM_OPT_VERSION
00502     //At this point, each Node object has received the simParameters,
00503     //parameters and the atom signatures info from the master Node
00504     //(proc 0). It's time to initialize the parallel IO manager and
00505     //read the binary per-atom file --Chao Mei
00506 
00507     //Step 1: initialize the parallel IO manager per Node
00508     ioMgr->initialize(this);
00509     #endif
00510 
00511   break;
00512 
00513   case 3:
00514 
00515     #ifdef MEM_OPT_VERSION
00516     //Step 2: read the binary per-atom files (signater index, coordinates etc.)
00517     ioMgr->readPerAtomInfo();
00518     #endif
00519 
00520   break;
00521 
00522   case 4:
00523 
00524     #ifdef MEM_OPT_VERSION
00525     //Step 3: update counters of tuples and exclusions inside Molecule object
00526     ioMgr->updateMolInfo();
00527 
00528     //Step 4: prepare distributing the atoms to neighboring procs if necessary
00529     ioMgr->migrateAtomsMGrp();
00530 
00531     //step 5: initialize patchMap and send it to every other processors
00532     //to decide atoms to patch distribution on every input processor
00533     if(!CkMyPe()) {
00534         workDistrib->patchMapInit(); // create space division
00535         workDistrib->sendPatchMap();
00536     }
00537     #endif
00538 
00539     #if USE_HPM
00540     HPM_Init(localRankOnNode);
00541     #endif    
00542 
00543     // take care of inital thread setting
00544     threadInit();
00545 
00546     // create blank AtomMap
00547     AtomMap::Object()->allocateMap(molecule->numAtoms);
00548 
00549     if (!CkMyPe()) {
00550       if (simParameters->useOptPME)
00551         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_OptPmeMgr::ckNew();
00552       else 
00553 #ifdef NAMD_CUDA
00554       if (simParameters->usePMECUDA) {
00555         // computePmeCUDAMgr was created in BackEnd.C
00556         // This empty branch is to avoid initializing ComputePmeMgr
00557       } else
00558 #endif
00559       if (simParameters->PMEOn) {
00560         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
00561       }
00562         #ifdef OPENATOM_VERSION
00563         if ( simParameters->openatomOn ) { 
00564           CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
00565         }
00566         #endif // OPENATOM_VERSION
00567 
00568     }
00569     
00570     #ifdef OPENATOM_VERSION
00571     if ( simParameters->openatomOn ) {
00572       // if ( ! CkMyPe() ) { 
00573         CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle); 
00574         startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
00575       // }
00576     }
00577     #endif // OPENATOM_VERSION
00578   
00579     // BEGIN LA
00580     rand = new Random(simParameters->randomSeed);
00581     rand->split(CkMyPe(), CkNumPes());
00582     // END LA
00583 
00584   break;
00585 
00586   case 5:
00587     #ifdef MEM_OPT_VERSION
00588     //Now, every input proc has received all the atoms necessary
00589     //to decide the patches those atoms belong to
00590     
00591     //step 1: integrate the migrated atoms into the atom list that
00592     //contains the initally distributed atoms, and sort the atoms
00593     //based on hydrogenList value
00594     ioMgr->integrateMigratedAtoms();
00595 
00596     //step 2: integrate the cluster size of each atom on each output proc
00597     ioMgr->integrateClusterSize();
00598 
00599     //step 3: calculate the number of atoms in each patch on every
00600     //input procs (atoms belonging to a patch may lie on different
00601     //procs), and reduce such info on proc 0. Such info is required
00602     //for determing which node a particular patch is assigned to.
00603     ioMgr->calcAtomsInEachPatch();
00604 
00605     //set to false to re-send PatchMap later
00606     workDistrib->setPatchMapArrived(false);
00607     #endif
00608     break;
00609   case 6:     
00610     if(simParameters->isSendSpanningTreeOn()) {                         
00611                         ProxyMgr::Object()->setSendSpanning();
00612     }
00613     if(simParameters->isRecvSpanningTreeOn()) {                         
00614                         ProxyMgr::Object()->setRecvSpanning();
00615     }
00616     if(simParameters->proxyTreeBranchFactor) {
00617                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);
00618     }
00619     #ifdef PROCTRACE_DEBUG
00620     DebugFileTrace::Instance("procTrace");
00621     #endif
00622 
00623     if (!CkMyPe()) {
00624       output = new Output; // create output object just on PE(0)
00625 
00626       #ifndef MEM_OPT_VERSION
00627       workDistrib->patchMapInit(); // create space division
00628       workDistrib->createHomePatches(); // load atoms into HomePatch(es)
00629       #endif
00630       
00631       workDistrib->assignNodeToPatch();
00632       workDistrib->mapComputes();         
00633       //ComputeMap::Object()->printComputeMap();
00634 
00635       // For MIC runs, take the additional step after the compute map has been created to
00636       //   assign the various computes to either the host or the device.  This info will
00637       //   be distributed across the PEs.
00638       #if defined(NAMD_MIC)
00639         mic_initHostDeviceLDB();
00640       #endif
00641           
00642           if(simParameters->simulateInitialMapping) {
00643           iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
00644                           << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
00645                   outputPatchComputeMaps("init_mapping", 0);
00646                   iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
00647                   BackEnd::exit();
00648           }
00649 
00650       registerUserEventsForAllComputeObjs();
00651 
00652       //in MEM_OPT_VERSION, patchMap is resent
00653       //because they have been updated since creation including
00654       //#atoms per patch, the proc a patch should stay etc. --Chao Mei
00655       workDistrib->sendPatchMap();
00656       #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00657       CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00658       //a node broadcast
00659       npm.createProxyInfo(PatchMap::Object()->numPatches());
00660       #endif
00661     }
00662     {
00663         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00664         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00665         if(CkMyRank()==0) {
00666             //just need to register once
00667             npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
00668         }
00669         npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
00670         #endif
00671     }
00672   break;
00673 
00674   case 7:
00675 #ifdef CHARM_HAS_MSA
00676     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00677       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00678       msm[CkMyPe()].initialize(new CkQdMsg);
00679     }
00680 #else
00681     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00682       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00683       MsmInitMsg *msg = new MsmInitMsg;
00684       Lattice lattice = simParameters->lattice;  // system lattice vectors
00685       ScaledPosition smin=0, smax=0;
00686       if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
00687         msg->smin = smin;
00688         msg->smax = smax;
00689         msm[CkMyPe()].initialize(msg);  // call from my own PE
00690       }
00691       else if ( ! CkMyPe() ) {
00692         pdb->get_extremes(smin, smax);  // only available on PE 0
00693         msg->smin = smin;
00694         msg->smax = smax;
00695         msm.initialize(msg);  // broadcast to chare group
00696       }
00697 
00698       /*
00699       CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00700       Node *node = nd.ckLocalBranch();
00701       ScaledPosition smin, smax;
00702       node->pdb->get_extremes(smin, smax);
00703       msg->smin = smin;                       // extreme positions in system
00704       msg->smax = smax;
00705       msm[CkMyPe()].initialize(msg);
00706       */
00707     }
00708 #endif
00709 
00710     if ( simParameters->PMEOn ) {
00711       if ( simParameters->useOptPME ) {
00712         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00713         pme[CkMyPe()].initialize(new CkQdMsg);
00714       }
00715       else {
00716         #ifdef OPENATOM_VERSION
00717         if ( simParameters->openatomOn ) { 
00718           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00719           moa[CkMyPe()].initialize(new CkQdMsg);
00720         }
00721         #endif // OPENATOM_VERSION
00722 #ifdef NAMD_CUDA
00723         if ( simParameters->usePMECUDA ) {
00724           if(CkMyRank()==0) {
00725             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00726             pme.ckLocalBranch()->initialize(new CkQdMsg);  // must run on pe 0 to call ckNew
00727           }
00728         } else 
00729 #endif
00730         {
00731           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00732           pme[CkMyPe()].initialize(new CkQdMsg);          
00733         }
00734       }
00735     }
00736     break;
00737 
00738   case 8:
00739 #ifdef NAMD_CUDA
00740     if ( (simParameters->useCUDA2 || simParameters->bondedCUDA) && CkMyRank()==0 ) {
00741       CProxy_ComputeCUDAMgr nb(CkpvAccess(BOCclass_group).computeCUDAMgr);
00742       nb.ckLocalBranch()->initialize(new CkQdMsg);
00743     }
00744 #endif
00745     break;
00746 
00747   case 9:
00748     workDistrib->sendComputeMap();
00749     break;
00750 
00751   case 10:
00752     #ifdef MEM_OPT_VERSION
00753     //migrate atoms to HomePatch processors
00754     ioMgr->sendAtomsToHomePatchProcs();
00755     #endif
00756     break;
00757     
00758   case 11:
00759     // part 2 of MSM init
00760     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00761       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00762       msm[CkMyPe()].initialize_create();  // call from my own PE
00763     }
00764 
00765     if ( simParameters->PMEOn ) {
00766       if ( simParameters->useOptPME ) {
00767         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00768         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00769       }
00770       else {
00771         #ifdef OPENATOM_VERSION
00772         if ( simParameters->openatomOn ) { 
00773           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00774           moa[CkMyPe()].initWorkers(new CkQdMsg);
00775         }
00776         #endif // OPENATOM_VERSION
00777 #ifdef NAMD_CUDA
00778         if ( simParameters->usePMECUDA ) {
00779           if(CkMyRank()==0) {
00780             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00781             pme[CkMyNode()].initialize_pencils(new CkQdMsg);
00782           }
00783         } else
00784 #endif
00785         {
00786           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00787           pme[CkMyPe()].initialize_pencils(new CkQdMsg);          
00788         }
00789       }
00790     }
00791 #ifdef CHARM_HAS_MSA
00792     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00793       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00794       msm[CkMyPe()].initWorkers(new CkQdMsg);
00795     }
00796 #else
00797     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00798       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00799       msm[CkMyPe()].update(new CkQdMsg);
00800     }
00801 #endif
00802 
00803     #ifdef MEM_OPT_VERSION
00804     //Now every processor has all the atoms it needs to create the HomePatches.
00805     //The HomePatches are created in parallel on every home patch procs.
00806     ioMgr->createHomePatches();
00807     #else
00808     if (!CkMyPe()) {
00809       workDistrib->distributeHomePatches();          
00810     }
00811     #endif
00812   break;
00813 
00814   case 12:
00815     if ( simParameters->PMEOn ) {
00816       if ( simParameters->useOptPME ) {
00817         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00818         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00819       }
00820       else {
00821         #ifdef OPENATOM_VERSION
00822         if ( simParameters->openatomOn ) { 
00823           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00824           moa[CkMyPe()].startWorkers(new CkQdMsg);
00825         }
00826         #endif // OPENATOM_VERSION
00827 #ifdef NAMD_CUDA
00828         if ( simParameters->usePMECUDA ) {
00829           if(CkMyRank()==0) {
00830             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00831             pme[CkMyNode()].activate_pencils(new CkQdMsg);
00832           }
00833         } else
00834 #endif
00835         {
00836           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00837           pme[CkMyPe()].activate_pencils(new CkQdMsg);          
00838         }
00839       }
00840     }
00841 #ifdef CHARM_HAS_MSA
00842     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00843       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00844       msm[CkMyPe()].startWorkers(new CkQdMsg);
00845     }
00846 #else
00847     /*
00848     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00849       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00850       //msm[CkMyPe()].startWorkers(new CkQdMsg);
00851     }
00852     */
00853 #endif
00854 
00855     proxyMgr->createProxies();  // need Home patches before this
00856     if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
00857 
00858 #ifdef NAMD_TCL
00859     // TclInitSubsystems() has a race condition so we create one interp per node here
00860     if (CkMyPe() && CkMyNodeSize() > 1 && ! CkMyRank()) Tcl_DeleteInterp(Tcl_CreateInterp());
00861 #endif
00862 
00863 #ifdef USE_NODEPATCHMGR
00864         //at this point, PatchMap info has been recved on PEs. It is time to create
00865         //the home patch spanning tree for receiving proxy list info
00866         if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
00867                 if(CkMyRank()==0) {
00868                         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00869                         npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
00870                 }
00871         }
00872 #endif
00873 
00874   break;
00875 
00876   case 13:
00877 
00878     // DMK - DEBUG - If, in MIC runs, the debug option to dump all the compute maps to files
00879     //   for debugging/verification purposes has been enabled, have each PE do so now.
00880     #if defined(NAMD_MIC)
00881       mic_dumpHostDeviceComputeMap();
00882     #endif
00883 
00884     if (!CkMyPe()) {
00885       iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
00886            << " COMPUTE OBJECTS\n" << endi;
00887     }
00888     DebugM(4,"Creating Computes\n");
00889     computeMgr->createComputes(ComputeMap::Object());
00890     DebugM(4,"Building Sequencers\n");
00891     buildSequencers();
00892     DebugM(4,"Initializing LDB\n");
00893     LdbCoordinator::Object()->initialize(PatchMap::Object(),ComputeMap::Object());
00894   break;
00895 
00896   case 14:
00897     // computes may create proxies on the fly so put these in separate phase
00898     Sync::Object()->openSync();  // decide if to open local Sync 
00899     if (proxySendSpanning || proxyRecvSpanning ) proxyMgr->buildProxySpanningTree();
00900 #ifdef CMK_BALANCED_INJECTION_API
00901     if(CkMyRank() == 0){
00902       // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00903       ck_set_GNI_BIConfig(balancedInjectionLevel);
00904       // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00905     }
00906 #endif
00907 
00908   break;
00909 
00910   case 15:
00911     {
00912         //For debugging
00913         /*if(!CkMyPe()){
00914         FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
00915         dumpbench(dumpFile);
00916         NAMD_die("Normal execution\n");
00917         }*/
00918     }
00919     #ifdef MEM_OPT_VERSION
00920     //free space in the Molecule object that are not used anymore
00921     ioMgr->freeMolSpace();
00922     #endif
00923     gotoRun = true;
00924   break;
00925 
00926   default:
00927     NAMD_bug("Startup Phase has a bug - check case statement");
00928   break;
00929 
00930   }
00931 
00932   startupPhase++;
00933   if (!CkMyPe()) {
00934     if (!gotoRun) {
00935       CkStartQD(CkIndex_Node::startUp((CkQdMsg*)0),&thishandle);
00936     } else {
00937       Node::messageRun();
00938     }
00939   }
00940 }
00941 
00942 #ifdef OPENATOM_VERSION
00943 void Node::doneMoaStart()
00944 {
00945 #ifdef OPENATOM_VERSION_DEBUG
00946   CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
00947 #endif //OPENATOM_VERSION_DEBUG
00948 }
00949 #endif //OPENATOM_VERSION
00950 
00951 void Node::namdOneCommInit()
00952 {
00953   if (CkpvAccess(comm) == NULL) {
00954     CkpvAccess(comm) = new Communicate();
00955 #ifdef DPMTA
00956     pvmc_init();
00957 #endif
00958   }
00959 }
00960 
00961 // Namd 1.X style Send/Recv of simulation information
00962 
00963 void Node::namdOneRecv() {
00964   if ( CmiMyRank() ) return;
00965 
00966   MIStream *conv_msg;
00967 
00968   // Receive molecule and simulation parameter information
00969   simParameters = node_simParameters = new SimParameters;
00970   //****** BEGIN CHARMM/XPLOR type changes
00971   parameters = node_parameters = new Parameters();
00972   //****** END CHARMM/XPLOR type changes
00973   molecule = node_molecule = new Molecule(simParameters,parameters);
00974 
00975   DebugM(4, "Getting SimParameters\n");
00976   conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
00977   simParameters->receive_SimParameters(conv_msg);
00978 
00979   DebugM(4, "Getting Parameters\n");
00980   conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
00981   parameters->receive_Parameters(conv_msg);
00982 
00983   DebugM(4, "Getting Molecule\n");
00984   conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00985   // Modified by JLai -- 10.21.11
00986   molecule->receive_Molecule(conv_msg);
00987   if(simParameters->goForcesOn) {
00988     iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
00989     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00990     molecule->receive_GoMolecule(conv_msg);
00991   } 
00992   // End of modification
00993   DebugM(4, "Done Receiving\n");
00994 }
00995 
00996 void Node::namdOneSend() {
00997   node_simParameters = simParameters;
00998   node_parameters = parameters;
00999   node_molecule = molecule;
01000 
01001   MOStream *conv_msg;
01002   // I'm Pe(0) so I send what I know
01003   DebugM(4, "Sending SimParameters\n");  
01004   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
01005   simParameters->send_SimParameters(conv_msg);
01006 
01007   DebugM(4, "Sending Parameters\n");
01008   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
01009   parameters->send_Parameters(conv_msg);
01010 
01011   DebugM(4, "Sending Molecule\n");
01012   int bufSize = BUFSIZE;
01013   if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01014   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01015   // Modified by JLai -- 10.21.11
01016   molecule->send_Molecule(conv_msg);
01017   
01018   if(simParameters->goForcesOn) {
01019     iout << iINFO <<  "Master Node sending GoMolecule Information" << "\n" << endi;
01020     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01021     molecule->send_GoMolecule(conv_msg);
01022   } // End of modification
01023 }
01024 
01025 
01026 void Node::reloadStructure(const char *fname, const char *pdbname) {
01027   delete molecule;
01028   molecule = state->molecule = 0;
01029   delete pdb;
01030   pdb = state->pdb = 0;
01031   state->loadStructure(fname,pdbname,1);
01032   this->molecule = state->molecule;
01033   this->pdb = state->pdb;
01034   CProxy_Node nodeProxy(thisgroup);
01035   nodeProxy.resendMolecule();
01036 }
01037 
01038 
01039 void Node::resendMolecule() {
01040   if ( CmiMyRank() ) {
01041     return;
01042   }
01043   if ( CmiMyPe() == 0 ) {
01044     int bufSize = BUFSIZE;
01045     MOStream *conv_msg;
01046     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, bufSize);
01047     parameters->send_Parameters(conv_msg);
01048     if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01049     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01050     molecule->send_Molecule(conv_msg);
01051   } else {
01052     MIStream *conv_msg;
01053     delete parameters;
01054     parameters = new Parameters;
01055     conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
01056     parameters->receive_Parameters(conv_msg);
01057     delete molecule;
01058     molecule = new Molecule(simParameters,parameters);
01059     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
01060     molecule->receive_Molecule(conv_msg);
01061   }
01062   node_parameters = parameters;
01063   node_molecule = molecule;
01064   SimParameters::nonbonded_select();
01065   if ( simParameters->PMEOn ) SimParameters::pme_select();
01066   computeMgr->sendBuildCudaExclusions();
01067   CProxy_Node nodeProxy(thisgroup);
01068   for ( int i=0; i<CmiMyNodeSize(); ++i ) {
01069     nodeProxy[CmiMyPe()+i].resendMolecule2();
01070   }
01071 }
01072 
01073 void Node::resendMolecule2() {
01074   parameters = node_parameters;
01075   molecule = node_molecule;
01076   AtomMap::Object()->allocateMap(molecule->numAtoms);
01077 }
01078 
01079 
01080 // Initial thread setup
01081 
01082 void Node::threadInit() {
01083   // Thread initialization
01084   if (CthImplemented()) {
01085     CthSetStrategyDefault(CthSelf());
01086   } else {
01087     NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
01088   }
01089 }
01090 
01091 //
01092 void Node::buildSequencers() {
01093   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01094   ResizeArrayIter<HomePatchElem> ai(*hpl);
01095 
01096   // Controller object is only on Pe(0)
01097   if ( ! CkMyPe() ) {
01098     Controller *controller = new Controller(state);
01099     state->useController(controller);
01100   }
01101 
01102   // Assign Sequencer to all HomePatch(es)
01103   for (ai=ai.begin(); ai != ai.end(); ai++) {
01104     HomePatch *patch = (*ai).patch;
01105     Sequencer *sequencer = new Sequencer(patch);
01106     patch->useSequencer(sequencer);
01107   }
01108 }
01109 
01110 
01111 
01112 //-----------------------------------------------------------------------
01113 // Node run() - broadcast to all nodes
01114 //-----------------------------------------------------------------------
01115 void Node::messageRun() {
01116   (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
01117 }
01118 
01119 
01120 //-----------------------------------------------------------------------
01121 // run(void) runs the specified simulation for the specified number of
01122 // steps, overriding the contents of the configuration file
01123 //-----------------------------------------------------------------------
01124 void Node::run()
01125 {
01126   // Start Controller (aka scalar Sequencer) on Pe(0)
01127 //  printf("\n\n I am in Node.C in run method about to call  state->runController\n\n");
01128   if ( ! CkMyPe() ) {
01129     state->runController();
01130   }
01131 
01132   DebugM(4, "Starting Sequencers\n");
01133   // Run Sequencer on each HomePatch - i.e. start simulation
01134   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01135   ResizeArrayIter<HomePatchElem> ai(*hpl);
01136   for (ai=ai.begin(); ai != ai.end(); ai++) {
01137     HomePatch *patch = (*ai).patch;
01138 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
01139     patch->runSequencer();
01140   }
01141 
01142   if (!CkMyPe()) {
01143     double newTime = CmiWallTimer();
01144     iout << iINFO << "Startup phase " << startupPhase-1 << " took "
01145          << newTime - startupTime << " s, "
01146          << memusage_MB() << " MB of memory in use\n";
01147     iout << iINFO << "Finished startup at " << newTime << " s, "
01148          << memusage_MB() << " MB of memory in use\n\n" << endi;
01149     fflush(stdout);
01150   }
01151   
01152 }
01153 
01154 
01155 //-----------------------------------------------------------------------
01156 // Node scriptBarrier() - twiddle parameters with simulation halted
01157 //-----------------------------------------------------------------------
01158 
01159 void Node::enableScriptBarrier() {
01160   CkStartQD(CkIndex_Node::scriptBarrier((CkQdMsg*)0),&thishandle);
01161 }
01162 
01163 void Node::scriptBarrier(CkQdMsg *qmsg) {
01164   delete qmsg;
01165   //script->awaken();
01166 }
01167 
01168 void Node::scriptParam(ScriptParamMsg *msg) {
01169   simParameters->scriptSet(msg->param,msg->value);
01170   delete msg;
01171 }
01172 
01173 void Node::reloadCharges(const char *filename) {
01174   FILE *file = fopen(filename,"r");
01175   if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
01176 
01177   int n = molecule->numAtoms;
01178   float *charge = new float[n];
01179 
01180   for ( int i = 0; i < n; ++i ) {
01181     if ( ! fscanf(file,"%f",&charge[i]) )
01182       NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
01183   }
01184 
01185   fclose(file);
01186   CProxy_Node(thisgroup).reloadCharges(charge,n);
01187   delete [] charge;
01188 }
01189 
01190 void Node::reloadCharges(float charge[], int n) {
01191   molecule->reloadCharges(charge,n);
01192 }
01193 
01194 
01195 // BEGIN gf
01196 void Node::reloadGridforceGrid(const char * key) {
01197     DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
01198     
01199     int gridnum;
01200     MGridforceParams *mgridParams;
01201     if (key == NULL) {
01202         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01203         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01204     } else {
01205         gridnum = simParameters->mgridforcelist.index_for_key(key);
01206         mgridParams = simParameters->mgridforcelist.find_key(key);
01207     }
01208     
01209     if (gridnum < 0 || mgridParams == NULL) {
01210         NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
01211     }
01212     
01213     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01214     if (grid == NULL) {
01215         NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
01216     }
01217     grid->reinitialize(simParameters, mgridParams);
01218     
01219     CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
01220     
01221     DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
01222 }
01223 
01224 void Node::updateGridScale(const char* key, Vector scale) {
01225     DebugM(4, "updateGridScale(char*, Vector) called on node " << CkMyPe() << "\n" << endi);
01226     
01227     int gridnum;
01228     MGridforceParams* mgridParams;
01229     if (key == NULL) {
01230         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01231         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01232     } else {
01233         gridnum = simParameters->mgridforcelist.index_for_key(key);
01234         mgridParams = simParameters->mgridforcelist.find_key(key);
01235     }
01236 
01237     if (gridnum < 0 || mgridParams == NULL) {
01238         NAMD_die("Node::updateGridScale(char*, Vector): Could not find grid.");
01239     }
01240     
01241     GridforceGrid* grid = molecule->get_gridfrc_grid(gridnum);
01242     if (grid == NULL) {
01243         NAMD_bug("Node::updateGridScale(char*, Vector): grid not found");
01244     }
01245     CProxy_Node(thisgroup).updateGridScale(gridnum, scale.x, scale.y, scale.z);
01246     
01247     DebugM(4, "updateGridScale(char*, Vector) finished\n" << endi);
01248 }
01249 void Node::updateGridScale(int gridnum, float sx, float sy, float sz) {
01250     if (CmiMyRank()) return;
01251     DebugM(4, "updateGridScale(char*, int, float, float, float) called on node " << CkMyPe() << "\n" << endi);
01252        
01253     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01254     if (grid == NULL) {
01255         NAMD_bug("Node::updateGridScale(char*, int, float, float, float):grid not found");
01256     }
01257     
01258     Vector scale(sx,sy,sz);
01259     simParameters->mgridforcelist.at_index(gridnum)->gridforceScale = scale;
01260     grid->set_scale( scale );
01261 
01262     DebugM(4, "updateGridScale(char*, int, float, float, float) finished\n" << endi);
01263 }
01264 
01265 void Node::reloadGridforceGrid(int gridnum) {
01266     if (CmiMyRank()) return;
01267     DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
01268     
01269     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01270     if (grid == NULL) {
01271         NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
01272     }
01273     
01274     if (CkMyPe()) {
01275         // not node 0 -> receive grid
01276         DebugM(4, "Receiving grid\n");
01277         
01278         delete grid;
01279         
01280         MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
01281         grid = GridforceGrid::unpack_grid(gridnum, msg);
01282         molecule->set_gridfrc_grid(gridnum, grid);
01283         delete msg;
01284     } else {
01285         // node 0 -> send grid
01286         DebugM(4, "Sending grid\n");
01287         
01288         MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
01289         GridforceGrid::pack_grid(grid, msg);
01290         msg->end();
01291         delete msg;
01292     }
01293     
01294     DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
01295 }
01296 // END gf
01297 
01298 
01299 // initiating replica
01300 void Node::sendCheckpointReq(int remote, const char *key, int task, Lattice &lat, ControllerState &cs) {
01301   CheckpointMsg *msg = new (1+strlen(key),0) CheckpointMsg;
01302   msg->replica = CmiMyPartition();
01303   msg->task = task;
01304   msg->checkpoint.lattice = lat;
01305   msg->checkpoint.state = cs;
01306   strcpy(msg->key,key);
01307   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01308   CmiSetHandler(env,recvCheckpointCReq_index);
01309 #if CMK_HAS_PARTITION
01310   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01311 #else
01312   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01313 #endif
01314 }
01315 
01316 // responding replica
01317 extern "C" {
01318   void recvCheckpointCReq_handler(envelope *env) {
01319     Node::Object()->recvCheckpointReq(CheckpointMsg::unpack(EnvToUsr(env)));
01320   }
01321 }
01322 
01323 // responding replica
01324 void Node::recvCheckpointReq(CheckpointMsg *msg) {
01325   state->controller->recvCheckpointReq(msg->key,msg->task,msg->checkpoint);
01326 
01327   int remote = msg->replica;
01328   msg->replica = CmiMyPartition();
01329   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01330   CmiSetHandler(env,recvCheckpointCAck_index);
01331 #if CMK_HAS_PARTITION
01332   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01333 #else
01334   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01335 #endif
01336 }
01337 
01338 // initiating replica
01339 extern "C" {
01340   void recvCheckpointCAck_handler(envelope *env) {
01341     Node::Object()->recvCheckpointAck(CheckpointMsg::unpack(EnvToUsr(env)));
01342   }
01343 }
01344 
01345 // initiating replica
01346 void Node::recvCheckpointAck(CheckpointMsg *msg) {
01347   state->controller->recvCheckpointAck(msg->checkpoint);
01348   delete msg;
01349 }
01350 
01351 
01352 void Node::sendEnableExitScheduler(void) {
01353   //CmiPrintf("sendEnableExitScheduler\n");
01354   CkQdMsg *msg = new CkQdMsg;
01355   CProxy_Node nodeProxy(thisgroup);
01356   nodeProxy[0].recvEnableExitScheduler(msg);
01357 }
01358 
01359 void Node::recvEnableExitScheduler(CkQdMsg *msg) {
01360   //CmiPrintf("recvEnableExitScheduler\n");
01361   delete msg;
01362   enableExitScheduler();
01363 }
01364 
01365 void Node::enableExitScheduler(void) {
01366   if ( CkMyPe() ) {
01367     sendEnableExitScheduler();
01368   } else {
01369     CkStartQD(CkIndex_Node::exitScheduler((CkQdMsg*)0),&thishandle);
01370   }
01371 }
01372 
01373 void Node::exitScheduler(CkQdMsg *msg) {
01374   //CmiPrintf("exitScheduler %d\n",CkMyPe());
01375   CsdExitScheduler();
01376   delete msg;
01377 }
01378 
01379 void Node::sendEnableEarlyExit(void) {
01380   CkQdMsg *msg = new CkQdMsg;
01381   CProxy_Node nodeProxy(thisgroup);
01382   nodeProxy[0].recvEnableEarlyExit(msg);
01383 }
01384 
01385 void Node::recvEnableEarlyExit(CkQdMsg *msg) {
01386   delete msg;
01387   enableEarlyExit();
01388 }
01389 
01390 void Node::enableEarlyExit(void) {
01391   if ( CkMyPe() ) {
01392     sendEnableEarlyExit();
01393   } else {
01394     CkStartQD(CkIndex_Node::earlyExit((CkQdMsg*)0),&thishandle);
01395   }
01396 }
01397 
01398 void Node::earlyExit(CkQdMsg *msg) {
01399   iout << iERROR << "Exiting prematurely; see error messages above.\n" << endi;
01400   if ( CmiNumPartitions() > 1 ) NAMD_quit("Exiting prematurely; see error messages above.");
01401   BackEnd::exit();
01402   delete msg;
01403 }
01404 
01405 
01406 //------------------------------------------------------------------------
01407 // Some odd utilities
01408 //------------------------------------------------------------------------
01409 void Node::saveMolDataPointers(NamdState *state)
01410 {
01411   this->molecule = state->molecule;
01412   this->parameters = state->parameters;
01413   this->simParameters = state->simParameters;
01414   this->configList = state->configList;
01415   this->pdb = state->pdb;
01416   this->state = state;
01417 }
01418 
01419 // entry methods for BG/P HPM (performance counters) library
01420 void Node::startHPM() {
01421 #if USE_HPM
01422   HPM_Start("500 steps", localRankOnNode);
01423 #endif
01424 }
01425 
01426 void Node::stopHPM() {
01427 #if USE_HPM
01428   HPM_Stop("500 steps", localRankOnNode);
01429   HPM_Print(CkMyPe(), localRankOnNode);
01430 #endif
01431 }
01432 
01433 void Node::traceBarrier(int turnOnTrace, int step){
01434         curTimeStep = step;
01435         if(turnOnTrace) traceBegin();
01436         else traceEnd();
01437 
01438     if(turnOnTrace) CmiTurnOnStats();
01439     else CmiTurnOffStats();
01440 
01441         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());    
01442         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01443         CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
01444         contribute(0, NULL, CkReduction::sum_int, cb);
01445         
01446 }
01447 
01448 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
01449         CmiAssert(CmiMyPe()==0);
01450         delete msg;     
01451         state->controller->resumeAfterTraceBarrier(curTimeStep);
01452 }
01453 
01454 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
01455 #ifdef MEASURE_NAMD_WITH_PAPI
01456         curMFlopStep = step;
01457         double results[NUM_PAPI_EVENTS+1];
01458 
01459         if(turnOnMeasure){              
01460           CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]=CmiWallTimer();
01461 
01462           long long counters[NUM_PAPI_EVENTS+1];
01463           int ret=PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
01464           if(ret==PAPI_OK)
01465             {
01466               //              CkPrintf("traceBarrier start counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01467             }
01468           else
01469             {
01470               CkPrintf("error PAPI_start_counters (%d) at step %d called on proc %d\n",ret , step, CkMyPe());
01471             }
01472           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)!=PAPI_OK)
01473             {
01474               CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01475             };
01476         }else{
01477           long long counters[NUM_PAPI_EVENTS+1];
01478           for(int i=0;i<NUM_PAPI_EVENTS;i++)  counters[i]=0LL;
01479           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)==PAPI_OK)
01480             {
01481 #if !MEASURE_PAPI_SPP
01482               results[0] = (double)counters[0]/1e6;
01483               results[1] = (double)counters[1]/1e6;
01484 #else
01485               for(int i=0;i<NUM_PAPI_EVENTS;i++)  results[i] = counters[i]/1e6;
01486 #endif
01487               //              for(int i=0;i<NUM_PAPI_EVENTS;i++) CkPrintf("[%d] counter %d is %ld\n",CkMyPe(),i,counters[i]);
01488             }
01489           else
01490             {
01491               //              CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01492             }
01493           //      CkPrintf("traceBarrier stop counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01494                 
01495           PAPI_stop_counters(counters, NUM_PAPI_EVENTS);        
01496         }
01497         if(CkMyPe()==0)
01498           //        CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01499         results[NUM_PAPI_EVENTS]=CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]; //starttime
01500         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01501         CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
01502         contribute(sizeof(double)*(NUM_PAPI_EVENTS+1), &results, CkReduction::sum_double, cb);  
01503 #endif
01504 }
01505 
01506 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
01507 #ifdef MEASURE_NAMD_WITH_PAPI
01508   
01509         if(simParameters->papiMeasureStartStep != curMFlopStep) {
01510                 double *results = (double *)msg->getData();
01511                 double endtime=CmiWallTimer();
01512                 int bstep = simParameters->papiMeasureStartStep;
01513                 int estep = bstep + simParameters->numPapiMeasureSteps;
01514 #if MEASURE_PAPI_SPP
01515                 CkPrintf("SPP INFO: PAPI_FP_OPS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[0]);
01516                 CkPrintf("SPP INFO: PAPI_TOT_INS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[1]);
01517                 CkPrintf("SPP INFO: perf::PERF_COUNT_HW_CACHE_LL:MISS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[2]);
01518                 CkPrintf("SPP INFO: DATA_PREFETCHER:ALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[3]);
01519                 CkPrintf("SPP INFO: PAPI_L1_DCA timestep %d to %d is %lf(1e6)\n", bstep,estep,results[4]);
01520                 CkPrintf("SPP INFO: PAPI_TOT_CYC timestep %d to % is %lf(1e6)\n", bstep,estep,results[5]);
01521                 //              CkPrintf("SPP INFO: INSTRUCTION_FETCH_STALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[6]);
01522                 //              CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]/CkNumPes());
01523                 CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]);
01524                 CkPrintf("SPP INFO: endtime %lf avgtime %lf tottime %lf\n", endtime,results[NUM_PAPI_EVENTS]/CkNumPes(),results[NUM_PAPI_EVENTS] );
01525 #else
01526                 if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
01527                         double totalFPIns = results[0];
01528                         if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
01529                         CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n", 
01530                                          bstep, estep, totalFPIns/CkNumPes());
01531                 }else{
01532                         char nameBuf[PAPI_MAX_STR_LEN];
01533                         CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ", 
01534                                          bstep, estep);
01535                         for(int i=0; i<NUM_PAPI_EVENTS; i++) {
01536                                 PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
01537                                 CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
01538                         }
01539                         CkPrintf("per processor\n");
01540                 }               
01541 #endif
01542         }
01543         delete msg;     
01544         state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
01545 #endif
01546 }
01547 
01548 extern char *gNAMDBinaryName;
01549 void Node::outputPatchComputeMaps(const char *filename, int tag){
01550         if(!simParameters->outputMaps && !simParameters->simulateInitialMapping) return;
01551 
01552         int numpes = CkNumPes();
01553         int nodesize = CkMyNodeSize();
01554         if(simParameters->simulateInitialMapping) {
01555                 numpes = simParameters->simulatedPEs;
01556                 nodesize = simParameters->simulatedNodeSize;
01557         }
01558 
01559         char fname[128];
01560         sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
01561 
01562         FILE *fp = fopen(fname, "w");
01563         if(fp == NULL) {
01564                 NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
01565                 return;
01566         }
01567         PatchMap *pMap = PatchMap::Object();
01568         ComputeMap *cMap = ComputeMap::Object();
01569         int numPatches = pMap->numPatches();
01570         int numComputes = cMap->numComputes();
01571         fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes, 
01572                         pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
01573         //output PatchMap info
01574         for(int i=0; i<numPatches; i++) {
01575         #ifdef MEM_OPT_VERSION
01576                 fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
01577         #else
01578                 fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
01579         #endif
01580         }
01581 
01582         //output ComputeMap info
01583         for(int i=0; i<numComputes; i++) {              
01584                 fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));             
01585         }
01586 }
01587 
01588 
01589 //======================================================================
01590 // Private functions
01591 
01592 #include "Node.def.h"
01593 

Generated on Sat Jul 21 01:17:14 2018 for NAMD by  doxygen 1.4.7