Node.C

Go to the documentation of this file.
00001 
00007 /*
00008    Toplevel routines for initializing a Node for a simulation
00009    one Node per Pe (processor element).
00010 */
00011 
00012 #if !defined(WIN32) || defined(__CYGWIN__)
00013 #include <unistd.h>
00014 #endif
00015 #include "InfoStream.h"
00016 #include "Node.decl.h"
00017 #include "Node.h"
00018 #ifdef DPMTA
00019 #include <pvm3.h>
00020 #endif
00021 
00022 #include "ProcessorPrivate.h"
00023 
00024 #define MIN_DEBUG_LEVEL 3
00025 //#define DEBUGM
00026 #include "Debug.h"
00027 
00028 #include <stdio.h>
00029 #include <converse.h>
00030 #include "memusage.h"
00031 #include "IMDOutput.h"
00032 #include "Lattice.h"
00033 #include "ComputeMsmMsa.h"  // needed for MsmMsaData definition
00034 #include "ComputeMsm.h"     // needed for MsmInitMsg definition
00035 #include "main.decl.h"
00036 #include "main.h"
00037 #include "WorkDistrib.h"
00038 #include "PatchMgr.h"
00039 #include "Patch.h"
00040 #include "Compute.h"
00041 #include "ComputeMap.h"
00042 #include "ComputeMgr.h"
00043 #include "Molecule.h"
00044 #include "HomePatchList.h"
00045 #include "AtomMap.h"
00046 #include "Sequencer.h"
00047 #include "Controller.h"
00048 #include "NamdState.h"
00049 #include "Output.h"
00050 #include "ProxyMgr.h"
00051 #include "PatchMap.h"
00052 #include "PatchMap.inl"
00053 #include "Parameters.h"
00054 #include "SimParameters.h"
00055 #include "Communicate.h"
00056 #include "LdbCoordinator.h"
00057 #include "ScriptTcl.h"
00058 #include "ComputeMgr.decl.h"
00059 #include "ComputePmeMgr.decl.h"
00060 // #ifdef NAMD_CUDA
00061 #include "ComputeCUDAMgr.decl.h"
00062 #include "ComputePmeCUDAMgr.decl.h"
00063 #include "ComputePmeCUDAMgr.h"
00064 // #endif
00065 #include "ComputeGridForceMgr.decl.h"
00066 #include "OptPmeMgr.decl.h"
00067 #include "Sync.h"
00068 #include "BackEnd.h"
00069 #include "PDB.h"
00070 #include "packmsg.h"
00071 #include "CollectionMgr.decl.h"
00072 #include "ParallelIOMgr.decl.h"
00073 #include "Vector.h"
00074 // BEGIN LA
00075 #include "Random.h"
00076 // END LA
00077 
00078 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00079 extern "C" void CApplicationInit();
00080 #endif
00081 
00082 #include "DumpBench.h"
00083 
00084 class CheckpointMsg : public CMessage_CheckpointMsg {
00085 public:
00086   int task;
00087   int replica;
00088   Controller::checkpoint checkpoint;
00089   char *key;
00090 };
00091 
00092 extern "C" {
00093   void recvCheckpointCReq_handler(envelope*);
00094   void recvCheckpointCAck_handler(envelope*);
00095 }
00096 
00097 #ifdef CMK_BALANCED_INJECTION_API
00098 #include "ckBIconfig.h"
00099 #endif
00100 
00101 #include "CollectionMgr.h"
00102 #include "CollectionMaster.h"
00103 #include "CollectionMgr.decl.h"
00104 #include "CollectionMaster.decl.h"
00105 
00106 #if USE_HPM
00107 extern "C" void HPM_Init(int);
00108 extern "C" void HPM_Start(char *label, int);
00109 extern "C" void HPM_Stop(char *label, int);
00110 extern "C" void HPM_Print(int, int);
00111 #endif
00112 
00113 #if defined(NAMD_MIC)
00114   extern void mic_dumpHostDeviceComputeMap();
00115   extern void mic_initHostDeviceLDB();
00116 #endif
00117 
00118 #ifdef MEASURE_NAMD_WITH_PAPI
00119 #include "papi.h"
00120 #if CMK_SMP
00121 #include <pthread.h>
00122 #endif
00123 #define NUM_PAPI_EVENTS 6
00124 CkpvDeclare(int *, papiEvents);
00125 
00126 #define MEASURE_PAPI_SPP 1
00127 #define MEASURE_PAPI_CACHE 0
00128 #define MEASURE_PAPI_FLOPS 0
00129 
00130 static void namdInitPapiCounters(){
00131         if(CkMyRank()==0){
00132                 //only initialize per OS process (i.e. a charm node)
00133                 int retval = PAPI_library_init(PAPI_VER_CURRENT);
00134                 if(retval != PAPI_VER_CURRENT) {
00135                         if(CkMyPe()==0){
00136                                 NAMD_die("PAPI library is not compatitible!");
00137                         }
00138                 }
00139         #if CMK_SMP
00140                 //now only consider systems that are compatible with POSIX
00141                 if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
00142                         if(CkMyPe()==0){
00143                                 NAMD_die("Multi-thread mode in PAPI could not be initialized!");
00144                         }
00145                 }
00146         #endif
00147         }
00148         CkpvInitialize(int *, papiEvents);
00149         CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS+1];
00150 
00151 #if MEASURE_PAPI_CACHE
00152         if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
00153                 CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
00154         }else{
00155                 if(CkMyPe()==0){
00156                         CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");                     
00157                 }
00158                 //if not default to PAPI_TOT_INS
00159                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00160         }
00161 
00162         if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
00163                 CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
00164         }else{
00165                 //if not default to PAPI_TOT_CYC
00166                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00167         }       
00168 #elif MEASURE_PAPI_FLOPS
00169         if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
00170                 CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
00171         }else{
00172                 if(CkMyPe()==0){
00173                         CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
00174                 }
00175                 //if not default to PAPI_TOT_INS
00176                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00177         }
00178 
00179         if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
00180                 CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
00181         }else{
00182                 //if not default to PAPI_TOT_CYC
00183                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00184         }
00185 #elif MEASURE_PAPI_SPP
00186 /* for SPP we record these
00187 1) PAPI_FP_OPS
00188 2) PAPI_TOT_INS
00189 3) perf::PERF_COUNT_HW_CACHE_LL:MISS
00190 4) DATA_PREFETCHER:ALL
00191 5) PAPI_L1_DCA
00192 6) INSTRUCTION_FETCH_STALL
00193 7) PAPI_TOT_CYC, and 
00194 8) real (wall) time
00195 */
00196         int papiEventSet = PAPI_NULL; 
00197         if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
00198           CmiAbort("PAPI failed to create event set!\n");
00199         }
00200 
00201         if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
00202                 CkpvAccess(papiEvents)[0] = PAPI_FP_OPS;
00203         }else{
00204                 if(CkMyPe()==0){
00205                         CkAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
00206                 }
00207         }
00208         if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
00209                 CkpvAccess(papiEvents)[1] = PAPI_TOT_INS;
00210         }else{
00211                 if(CkMyPe()==0){
00212                         CkAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
00213                 }
00214         }
00215         int EventCode;
00216         int ret;
00217         ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
00218         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00219           CkpvAccess(papiEvents)[2] = EventCode;
00220         }else{
00221                 if(CkMyPe()==0){
00222                         CkAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
00223                 }
00224         }
00225         ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
00226         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00227           CkpvAccess(papiEvents)[3] = EventCode;
00228         }else{
00229                 if(CkMyPe()==0){
00230                         CkAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
00231                 }
00232         }
00233         if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
00234                 CkpvAccess(papiEvents)[4] = PAPI_L1_DCA;
00235         }else{
00236                 if(CkMyPe()==0){
00237                         CkAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
00238                 }
00239         }
00240         /*      ret=PAPI_event_name_to_code("INSTRUCTION_FETCH_STALL",&EventCode);
00241         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00242           CkpvAccess(papiEvents)[5] = EventCode;
00243         }else{
00244                 if(CkMyPe()==0){
00245                         CkAbort("WARNING: INSTRUCTION_FETCH_STALL doesn't exist on this platform!");
00246                 }
00247         }
00248         */
00249         if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
00250                 CkpvAccess(papiEvents)[5] = PAPI_TOT_CYC;
00251         }else{
00252                 if(CkMyPe()==0){
00253                         CkAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
00254                 }
00255         }
00256         for(int i=0;i<NUM_PAPI_EVENTS;i++)
00257           {
00258             int papiRetValue=PAPI_add_events(papiEventSet, &CkpvAccess(papiEvents)[i],1);
00259             if (papiRetValue != PAPI_OK) {
00260               CkPrintf("failure for event %d\n",i);
00261               if (papiRetValue == PAPI_ECNFLCT) {
00262                 CmiAbort("PAPI events conflict! Please re-assign event types!\n");
00263               } else {
00264                 CmiAbort("PAPI failed to add designated events!\n");
00265               }
00266             }
00267             
00268           }
00269 #endif
00270 }
00271 #endif
00272 
00273 #ifdef OPENATOM_VERSION
00274 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
00275 {
00276   CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
00277 }
00278 #endif //OPENATOM_VERSION
00279 
00280 //======================================================================
00281 // Public Functions
00282 
00283 //----------------------------------------------------------------------
00284 
00285 int eventEndOfTimeStep;
00286 double startupTime;
00287 
00288 //----------------------------------------------------------------------
00289 // BOC constructor
00290 Node::Node(GroupInitMsg *msg)
00291 {    
00292   DebugM(4,"Creating Node\n");
00293 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00294   CApplicationInit();
00295 #endif
00296   if (CkpvAccess(Node_instance) == 0) {
00297     CkpvAccess(Node_instance) = this;
00298     eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep");
00299   } else {
00300     NAMD_bug("Node::Node() - another instance of Node exists!");
00301   }
00302 
00303   CkpvAccess(BOCclass_group) = msg->group;
00304   delete msg;
00305 
00306   CkpvAccess(BOCclass_group).node = thisgroup;
00307 
00308   recvCheckpointCReq_index = CmiRegisterHandler((CmiHandler)recvCheckpointCReq_handler);
00309   recvCheckpointCAck_index = CmiRegisterHandler((CmiHandler)recvCheckpointCAck_handler);
00310 
00311   startupPhase = 0;
00312 
00313   molecule = NULL;
00314   parameters = NULL;
00315   simParameters = NULL;
00316   configList = NULL;
00317   pdb = NULL;
00318   state = NULL;
00319   output = NULL;
00320   imd = new IMDOutput;
00321   colvars = 0;
00322 
00323 #if USE_HPM
00324   // assumes that this will be done only on BG/P
00325   TopoManager *tmgr = new TopoManager();
00326   int x, y, z;
00327   tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
00328   delete tmgr;
00329 #endif
00330 
00331   specialTracing = traceAvailable() && (traceIsOn()==0);
00332 
00333   DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
00334   patchMap = PatchMap::Instance();
00335   atomMap = AtomMap::Instance();
00336   if ( CkMyRank() == 0 ) ComputeMap::Instance();
00337 
00338   //Note: Binding BOC vars such as workDistrib has been moved
00339   //to the 1st phase of startup because the in-order message delivery
00340   //is not always guaranteed --Chao Mei
00341 #ifdef CMK_BALANCED_INJECTION_API
00342   if(CkMyRank() == 0){
00343     balancedInjectionLevel=ck_get_GNI_BIConfig();
00344     // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00345     ck_set_GNI_BIConfig(20);
00346     // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00347   }
00348 #endif
00349 
00350 }
00351 
00352 //----------------------------------------------------------------------
00353 // ~Node(void) needs to clean up everything.
00354 
00355 Node::~Node(void)
00356 {
00357   delete output;
00358   delete computeMap;
00359   delete atomMap;
00360   delete patchMap;
00361   delete CkpvAccess(comm);
00362   // BEGIN LA
00363   delete rand;
00364   // END LA
00365 #ifdef MEASURE_NAMD_WITH_PAPI
00366   delete CkpvAccess(papiEvents);
00367 #endif
00368 }
00369 
00370 void Node::bindBocVars(){
00371     DebugM(4,"Binding to BOC's\n");
00372     CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00373     patchMgr = pm.ckLocalBranch();
00374     CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
00375     proxyMgr = prm.ckLocalBranch();
00376     CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
00377     workDistrib = wd.ckLocalBranch();
00378     CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00379     computeMgr = cm.ckLocalBranch();
00380     CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
00381     ldbCoordinator = lc.ckLocalBranch();
00382   #ifdef MEM_OPT_VERSION      
00383     CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
00384     ioMgr = io.ckLocalBranch();
00385   #endif
00386 
00387 }
00388 
00389 //----------------------------------------------------------------------
00390 // Malloc Test Sequence
00391 void Node::mallocTest(int step) {
00392   int MB = 1024*1024;
00393   int size = 100;
00394   char* foo = (char*) malloc(size*MB);
00395   if ( ! foo ) {
00396     char buf[256];
00397     sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
00398     NAMD_die(buf);
00399   }
00400   memset(foo,0,size*MB*sizeof(char));
00401 }
00402 
00403 void Node::mallocTestQd(CkQdMsg *qmsg) {
00404   delete qmsg;
00405   if ( mallocTest_size ) {
00406     CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
00407   } else {
00408     CkPrintf("Starting malloc test on all PEs.\n");
00409   }
00410   fflush(stdout);
00411   ++mallocTest_size;
00412   CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00413   (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
00414 }
00415 
00416 //----------------------------------------------------------------------
00417 // Startup Sequence
00418 
00419 void Node::messageStartUp() {
00420   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00421 }
00422 
00423 void Node::startUp(CkQdMsg *qmsg) {
00424   delete qmsg;
00425   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00426 }
00427 
00428 SimParameters *node_simParameters;
00429 Parameters *node_parameters;
00430 Molecule *node_molecule;
00431 
00432 extern void registerUserEventsForAllComputeObjs(void);
00433 
00434 void Node::startup() {
00435   int gotoRun = false;
00436   double newTime;
00437 
00438   if (!CkMyPe()) {
00439     if (!startupPhase) {
00440       iout << iINFO << "\n";
00441       startupTime = CmiWallTimer();
00442       iout << iINFO << "Entering startup at " << startupTime << " s, ";
00443     } else {
00444       newTime = CmiWallTimer();
00445       iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00446            << newTime - startupTime << " s, ";
00447       startupTime = newTime;
00448     }
00449     iout << memusage_MB() << " MB of memory in use\n" << endi;
00450     fflush(stdout);
00451   }
00452   switch (startupPhase) {
00453 
00454   case 0:
00455     computeMap = ComputeMap::Object();
00456     namdOneCommInit(); // Namd1.X style
00457   break;
00458 
00459   case 1:
00460       bindBocVars();
00461 
00462     // send & receive molecule, simparameters... (Namd1.X style)
00463     if (CkMyPe()) {
00464       namdOneRecv();
00465     } else {
00466       namdOneSend();
00467     }
00468   break;
00469 
00470   case 2:
00471     // fix up one-per-node objects (for SMP version)
00472     simParameters = node_simParameters;
00473     parameters = node_parameters;
00474     molecule = node_molecule;
00475 
00476     SimParameters::nonbonded_select();
00477     
00478     #if !CMK_SMP || ! USE_CKLOOP
00479     //the CkLoop library should be only used in SMP mode
00480     simParameters->useCkLoop = 0;
00481     #else
00482     if ( CkNumPes() < 2 * CkNumNodes() ) simParameters->useCkLoop = 0;
00483     #endif
00484 
00485 
00486     if ( simParameters->mallocTest ) {
00487       if (!CkMyPe()) {
00488         mallocTest_size = 0;
00489         CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00490       }
00491       return;
00492     }
00493 
00494       
00495         #ifdef MEASURE_NAMD_WITH_PAPI
00496         if(simParameters->papiMeasure) namdInitPapiCounters();  
00497         #endif
00498     
00499     #ifdef MEM_OPT_VERSION
00500     //At this point, each Node object has received the simParameters,
00501     //parameters and the atom signatures info from the master Node
00502     //(proc 0). It's time to initialize the parallel IO manager and
00503     //read the binary per-atom file --Chao Mei
00504 
00505     //Step 1: initialize the parallel IO manager per Node
00506     ioMgr->initialize(this);
00507     #endif
00508 
00509   break;
00510 
00511   case 3:
00512 
00513     #ifdef MEM_OPT_VERSION
00514     //Step 2: read the binary per-atom files (signater index, coordinates etc.)
00515     ioMgr->readPerAtomInfo();
00516     #endif
00517 
00518   break;
00519 
00520   case 4:
00521 
00522     #ifdef MEM_OPT_VERSION
00523     //Step 3: update counters of tuples and exclusions inside Molecule object
00524     ioMgr->updateMolInfo();
00525 
00526     //Step 4: prepare distributing the atoms to neighboring procs if necessary
00527     ioMgr->migrateAtomsMGrp();
00528 
00529     //step 5: initialize patchMap and send it to every other processors
00530     //to decide atoms to patch distribution on every input processor
00531     if(!CkMyPe()) {
00532         workDistrib->patchMapInit(); // create space division
00533         workDistrib->sendPatchMap();
00534     }
00535     #endif
00536 
00537     #if USE_HPM
00538     HPM_Init(localRankOnNode);
00539     #endif    
00540 
00541     // take care of inital thread setting
00542     threadInit();
00543 
00544     // create blank AtomMap
00545     AtomMap::Object()->allocateMap(molecule->numAtoms);
00546 
00547     if (!CkMyPe()) {
00548       if (simParameters->useOptPME)
00549         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_OptPmeMgr::ckNew();
00550       else 
00551 #ifdef NAMD_CUDA
00552       if (simParameters->usePMECUDA) {
00553         // computePmeCUDAMgr was created in BackEnd.C
00554         // This empty branch is to avoid initializing ComputePmeMgr
00555       } else
00556 #endif
00557       if (simParameters->PMEOn) {
00558         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
00559       }
00560         #ifdef OPENATOM_VERSION
00561         if ( simParameters->openatomOn ) { 
00562           CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
00563         }
00564         #endif // OPENATOM_VERSION
00565 
00566     }
00567     
00568     #ifdef OPENATOM_VERSION
00569     if ( simParameters->openatomOn ) {
00570       // if ( ! CkMyPe() ) { 
00571         CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle); 
00572         startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
00573       // }
00574     }
00575     #endif // OPENATOM_VERSION
00576   
00577     // BEGIN LA
00578     rand = new Random(simParameters->randomSeed);
00579     rand->split(CkMyPe(), CkNumPes());
00580     // END LA
00581 
00582   break;
00583 
00584   case 5:
00585     #ifdef MEM_OPT_VERSION
00586     //Now, every input proc has received all the atoms necessary
00587     //to decide the patches those atoms belong to
00588     
00589     //step 1: integrate the migrated atoms into the atom list that
00590     //contains the initally distributed atoms, and sort the atoms
00591     //based on hydrogenList value
00592     ioMgr->integrateMigratedAtoms();
00593 
00594     //step 2: integrate the cluster size of each atom on each output proc
00595     ioMgr->integrateClusterSize();
00596 
00597     //step 3: calculate the number of atoms in each patch on every
00598     //input procs (atoms belonging to a patch may lie on different
00599     //procs), and reduce such info on proc 0. Such info is required
00600     //for determing which node a particular patch is assigned to.
00601     ioMgr->calcAtomsInEachPatch();
00602 
00603     //set to false to re-send PatchMap later
00604     workDistrib->setPatchMapArrived(false);
00605     #endif
00606     break;
00607   case 6:     
00608     if(simParameters->isSendSpanningTreeOn()) {                         
00609                         ProxyMgr::Object()->setSendSpanning();
00610     }
00611     if(simParameters->isRecvSpanningTreeOn()) {                         
00612                         ProxyMgr::Object()->setRecvSpanning();
00613     }
00614     if(simParameters->proxyTreeBranchFactor) {
00615                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);
00616     }
00617     #ifdef PROCTRACE_DEBUG
00618     DebugFileTrace::Instance("procTrace");
00619     #endif
00620 
00621     if (!CkMyPe()) {
00622       output = new Output; // create output object just on PE(0)
00623 
00624       #ifndef MEM_OPT_VERSION
00625       workDistrib->patchMapInit(); // create space division
00626       workDistrib->createHomePatches(); // load atoms into HomePatch(es)
00627       #endif
00628       
00629       workDistrib->assignNodeToPatch();
00630       workDistrib->mapComputes();         
00631       //ComputeMap::Object()->printComputeMap();
00632 
00633       // For MIC runs, take the additional step after the compute map has been created to
00634       //   assign the various computes to either the host or the device.  This info will
00635       //   be distributed across the PEs.
00636       #if defined(NAMD_MIC)
00637         mic_initHostDeviceLDB();
00638       #endif
00639           
00640           if(simParameters->simulateInitialMapping) {
00641           iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
00642                           << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
00643                   outputPatchComputeMaps("init_mapping", 0);
00644                   iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
00645                   BackEnd::exit();
00646           }
00647 
00648       registerUserEventsForAllComputeObjs();
00649 
00650       //in MEM_OPT_VERSION, patchMap is resent
00651       //because they have been updated since creation including
00652       //#atoms per patch, the proc a patch should stay etc. --Chao Mei
00653       workDistrib->sendPatchMap();
00654       #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00655       CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00656       //a node broadcast
00657       npm.createProxyInfo(PatchMap::Object()->numPatches());
00658       #endif
00659     }
00660     {
00661         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00662         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00663         if(CkMyRank()==0) {
00664             //just need to register once
00665             npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
00666         }
00667         npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
00668         #endif
00669     }
00670   break;
00671 
00672   case 7:
00673 #ifdef CHARM_HAS_MSA
00674     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00675       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00676       msm[CkMyPe()].initialize(new CkQdMsg);
00677     }
00678 #else
00679     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00680       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00681       MsmInitMsg *msg = new MsmInitMsg;
00682       Lattice lattice = simParameters->lattice;  // system lattice vectors
00683       ScaledPosition smin=0, smax=0;
00684       if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
00685         msg->smin = smin;
00686         msg->smax = smax;
00687         msm[CkMyPe()].initialize(msg);  // call from my own PE
00688       }
00689       else if ( ! CkMyPe() ) {
00690         pdb->get_extremes(smin, smax);  // only available on PE 0
00691         msg->smin = smin;
00692         msg->smax = smax;
00693         msm.initialize(msg);  // broadcast to chare group
00694       }
00695 
00696       /*
00697       CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00698       Node *node = nd.ckLocalBranch();
00699       ScaledPosition smin, smax;
00700       node->pdb->get_extremes(smin, smax);
00701       msg->smin = smin;                       // extreme positions in system
00702       msg->smax = smax;
00703       msm[CkMyPe()].initialize(msg);
00704       */
00705     }
00706 #endif
00707 
00708     if ( simParameters->PMEOn ) {
00709       if ( simParameters->useOptPME ) {
00710         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00711         pme[CkMyPe()].initialize(new CkQdMsg);
00712       }
00713       else {
00714         #ifdef OPENATOM_VERSION
00715         if ( simParameters->openatomOn ) { 
00716           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00717           moa[CkMyPe()].initialize(new CkQdMsg);
00718         }
00719         #endif // OPENATOM_VERSION
00720 #ifdef NAMD_CUDA
00721         if ( simParameters->usePMECUDA ) {
00722           if(CkMyRank()==0) {
00723             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00724             pme.ckLocalBranch()->initialize(new CkQdMsg);  // must run on pe 0 to call ckNew
00725           }
00726         } else 
00727 #endif
00728         {
00729           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00730           pme[CkMyPe()].initialize(new CkQdMsg);          
00731         }
00732       }
00733     }
00734     break;
00735 
00736   case 8:
00737 #ifdef NAMD_CUDA
00738     if ( (simParameters->useCUDA2 || simParameters->bondedCUDA) && CkMyRank()==0 ) {
00739       CProxy_ComputeCUDAMgr nb(CkpvAccess(BOCclass_group).computeCUDAMgr);
00740       nb[CkMyNode()].initialize(new CkQdMsg);
00741     }
00742 #endif
00743     break;
00744 
00745   case 9:
00746     workDistrib->sendComputeMap();
00747     break;
00748 
00749   case 10:
00750     #ifdef MEM_OPT_VERSION
00751     //migrate atoms to HomePatch processors
00752     ioMgr->sendAtomsToHomePatchProcs();
00753     #endif
00754     break;
00755     
00756   case 11:
00757     // part 2 of MSM init
00758     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00759       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00760       msm[CkMyPe()].initialize_create();  // call from my own PE
00761     }
00762 
00763     if ( simParameters->PMEOn ) {
00764       if ( simParameters->useOptPME ) {
00765         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00766         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00767       }
00768       else {
00769         #ifdef OPENATOM_VERSION
00770         if ( simParameters->openatomOn ) { 
00771           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00772           moa[CkMyPe()].initWorkers(new CkQdMsg);
00773         }
00774         #endif // OPENATOM_VERSION
00775 #ifdef NAMD_CUDA
00776         if ( simParameters->usePMECUDA ) {
00777           if(CkMyRank()==0) {
00778             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00779             pme[CkMyNode()].initialize_pencils(new CkQdMsg);
00780           }
00781         } else
00782 #endif
00783         {
00784           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00785           pme[CkMyPe()].initialize_pencils(new CkQdMsg);          
00786         }
00787       }
00788     }
00789 #ifdef CHARM_HAS_MSA
00790     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00791       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00792       msm[CkMyPe()].initWorkers(new CkQdMsg);
00793     }
00794 #else
00795     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00796       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00797       msm[CkMyPe()].update(new CkQdMsg);
00798     }
00799 #endif
00800 
00801     #ifdef MEM_OPT_VERSION
00802     //Now every processor has all the atoms it needs to create the HomePatches.
00803     //The HomePatches are created in parallel on every home patch procs.
00804     ioMgr->createHomePatches();
00805     #else
00806     if (!CkMyPe()) {
00807       workDistrib->distributeHomePatches();          
00808     }
00809     #endif
00810   break;
00811 
00812   case 12:
00813     if ( simParameters->PMEOn ) {
00814       if ( simParameters->useOptPME ) {
00815         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00816         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00817       }
00818       else {
00819         #ifdef OPENATOM_VERSION
00820         if ( simParameters->openatomOn ) { 
00821           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00822           moa[CkMyPe()].startWorkers(new CkQdMsg);
00823         }
00824         #endif // OPENATOM_VERSION
00825 #ifdef NAMD_CUDA
00826         if ( simParameters->usePMECUDA ) {
00827           if(CkMyRank()==0) {
00828             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00829             pme[CkMyNode()].activate_pencils(new CkQdMsg);
00830           }
00831         } else
00832 #endif
00833         {
00834           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00835           pme[CkMyPe()].activate_pencils(new CkQdMsg);          
00836         }
00837       }
00838     }
00839 #ifdef CHARM_HAS_MSA
00840     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00841       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00842       msm[CkMyPe()].startWorkers(new CkQdMsg);
00843     }
00844 #else
00845     /*
00846     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00847       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00848       //msm[CkMyPe()].startWorkers(new CkQdMsg);
00849     }
00850     */
00851 #endif
00852 
00853     proxyMgr->createProxies();  // need Home patches before this
00854     if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
00855 
00856 #ifdef NAMD_TCL
00857     // TclInitSubsystems() has a race condition so we create one interp per node here
00858     if (CkMyPe() && CkMyNodeSize() > 1 && ! CkMyRank()) Tcl_DeleteInterp(Tcl_CreateInterp());
00859 #endif
00860 
00861 #ifdef USE_NODEPATCHMGR
00862         //at this point, PatchMap info has been recved on PEs. It is time to create
00863         //the home patch spanning tree for receiving proxy list info
00864         if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
00865                 if(CkMyRank()==0) {
00866                         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00867                         npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
00868                 }
00869         }
00870 #endif
00871 
00872   break;
00873 
00874   case 13:
00875 
00876     // DMK - DEBUG - If, in MIC runs, the debug option to dump all the compute maps to files
00877     //   for debugging/verification purposes has been enabled, have each PE do so now.
00878     #if defined(NAMD_MIC)
00879       mic_dumpHostDeviceComputeMap();
00880     #endif
00881 
00882     if (!CkMyPe()) {
00883       iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
00884            << " COMPUTE OBJECTS\n" << endi;
00885     }
00886     DebugM(4,"Creating Computes\n");
00887     computeMgr->createComputes(ComputeMap::Object());
00888     DebugM(4,"Building Sequencers\n");
00889     buildSequencers();
00890     DebugM(4,"Initializing LDB\n");
00891     LdbCoordinator::Object()->initialize(PatchMap::Object(),ComputeMap::Object());
00892   break;
00893 
00894   case 14:
00895     // computes may create proxies on the fly so put these in separate phase
00896     Sync::Object()->openSync();  // decide if to open local Sync 
00897     if (proxySendSpanning || proxyRecvSpanning ) proxyMgr->buildProxySpanningTree();
00898 #ifdef CMK_BALANCED_INJECTION_API
00899     if(CkMyRank() == 0){
00900       // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00901       ck_set_GNI_BIConfig(balancedInjectionLevel);
00902       // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00903     }
00904 #endif
00905 
00906   break;
00907 
00908   case 15:
00909     {
00910         //For debugging
00911         /*if(!CkMyPe()){
00912         FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
00913         dumpbench(dumpFile);
00914         NAMD_die("Normal execution\n");
00915         }*/
00916     }
00917     #ifdef MEM_OPT_VERSION
00918     //free space in the Molecule object that are not used anymore
00919     ioMgr->freeMolSpace();
00920     #endif
00921     gotoRun = true;
00922   break;
00923 
00924   default:
00925     NAMD_bug("Startup Phase has a bug - check case statement");
00926   break;
00927 
00928   }
00929 
00930   startupPhase++;
00931   if (!CkMyPe()) {
00932     if (!gotoRun) {
00933       CkStartQD(CkIndex_Node::startUp((CkQdMsg*)0),&thishandle);
00934     } else {
00935       Node::messageRun();
00936     }
00937   }
00938 }
00939 
00940 #ifdef OPENATOM_VERSION
00941 void Node::doneMoaStart()
00942 {
00943 #ifdef OPENATOM_VERSION_DEBUG
00944   CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
00945 #endif //OPENATOM_VERSION_DEBUG
00946 }
00947 #endif //OPENATOM_VERSION
00948 
00949 void Node::namdOneCommInit()
00950 {
00951   if (CkpvAccess(comm) == NULL) {
00952     CkpvAccess(comm) = new Communicate();
00953 #ifdef DPMTA
00954     pvmc_init();
00955 #endif
00956   }
00957 }
00958 
00959 // Namd 1.X style Send/Recv of simulation information
00960 
00961 void Node::namdOneRecv() {
00962   if ( CmiMyRank() ) return;
00963 
00964   MIStream *conv_msg;
00965 
00966   // Receive molecule and simulation parameter information
00967   simParameters = node_simParameters = new SimParameters;
00968   //****** BEGIN CHARMM/XPLOR type changes
00969   parameters = node_parameters = new Parameters();
00970   //****** END CHARMM/XPLOR type changes
00971   molecule = node_molecule = new Molecule(simParameters,parameters);
00972 
00973   DebugM(4, "Getting SimParameters\n");
00974   conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
00975   simParameters->receive_SimParameters(conv_msg);
00976 
00977   DebugM(4, "Getting Parameters\n");
00978   conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
00979   parameters->receive_Parameters(conv_msg);
00980 
00981   DebugM(4, "Getting Molecule\n");
00982   conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00983   // Modified by JLai -- 10.21.11
00984   molecule->receive_Molecule(conv_msg);
00985   if(simParameters->goForcesOn) {
00986     iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
00987     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00988     molecule->receive_GoMolecule(conv_msg);
00989   } 
00990   // End of modification
00991   DebugM(4, "Done Receiving\n");
00992 }
00993 
00994 void Node::namdOneSend() {
00995   node_simParameters = simParameters;
00996   node_parameters = parameters;
00997   node_molecule = molecule;
00998 
00999   MOStream *conv_msg;
01000   // I'm Pe(0) so I send what I know
01001   DebugM(4, "Sending SimParameters\n");  
01002   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
01003   simParameters->send_SimParameters(conv_msg);
01004 
01005   DebugM(4, "Sending Parameters\n");
01006   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
01007   parameters->send_Parameters(conv_msg);
01008 
01009   DebugM(4, "Sending Molecule\n");
01010   int bufSize = BUFSIZE;
01011   if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01012   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01013   // Modified by JLai -- 10.21.11
01014   molecule->send_Molecule(conv_msg);
01015   
01016   if(simParameters->goForcesOn) {
01017     iout << iINFO <<  "Master Node sending GoMolecule Information" << "\n" << endi;
01018     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01019     molecule->send_GoMolecule(conv_msg);
01020   } // End of modification
01021 }
01022 
01023 
01024 void Node::reloadStructure(const char *fname, const char *pdbname) {
01025   delete molecule;
01026   molecule = state->molecule = 0;
01027   delete pdb;
01028   pdb = state->pdb = 0;
01029   state->loadStructure(fname,pdbname,1);
01030   this->molecule = state->molecule;
01031   this->pdb = state->pdb;
01032   CProxy_Node nodeProxy(thisgroup);
01033   nodeProxy.resendMolecule();
01034 }
01035 
01036 
01037 void Node::resendMolecule() {
01038   if ( CmiMyRank() ) {
01039     return;
01040   }
01041   if ( CmiMyPe() == 0 ) {
01042     int bufSize = BUFSIZE;
01043     MOStream *conv_msg;
01044     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, bufSize);
01045     parameters->send_Parameters(conv_msg);
01046     if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01047     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01048     molecule->send_Molecule(conv_msg);
01049   } else {
01050     MIStream *conv_msg;
01051     delete parameters;
01052     parameters = new Parameters;
01053     conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
01054     parameters->receive_Parameters(conv_msg);
01055     delete molecule;
01056     molecule = new Molecule(simParameters,parameters);
01057     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
01058     molecule->receive_Molecule(conv_msg);
01059   }
01060   node_parameters = parameters;
01061   node_molecule = molecule;
01062   SimParameters::nonbonded_select();
01063   computeMgr->sendBuildCudaExclusions();
01064   CProxy_Node nodeProxy(thisgroup);
01065   for ( int i=0; i<CmiMyNodeSize(); ++i ) {
01066     nodeProxy[CmiMyPe()+i].resendMolecule2();
01067   }
01068 }
01069 
01070 void Node::resendMolecule2() {
01071   parameters = node_parameters;
01072   molecule = node_molecule;
01073   AtomMap::Object()->allocateMap(molecule->numAtoms);
01074 }
01075 
01076 
01077 // Initial thread setup
01078 
01079 void Node::threadInit() {
01080   // Thread initialization
01081   if (CthImplemented()) {
01082     CthSetStrategyDefault(CthSelf());
01083   } else {
01084     NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
01085   }
01086 }
01087 
01088 //
01089 void Node::buildSequencers() {
01090   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01091   ResizeArrayIter<HomePatchElem> ai(*hpl);
01092 
01093   // Controller object is only on Pe(0)
01094   if ( ! CkMyPe() ) {
01095     Controller *controller = new Controller(state);
01096     state->useController(controller);
01097   }
01098 
01099   // Assign Sequencer to all HomePatch(es)
01100   for (ai=ai.begin(); ai != ai.end(); ai++) {
01101     HomePatch *patch = (*ai).patch;
01102     Sequencer *sequencer = new Sequencer(patch);
01103     patch->useSequencer(sequencer);
01104   }
01105 }
01106 
01107 
01108 
01109 //-----------------------------------------------------------------------
01110 // Node run() - broadcast to all nodes
01111 //-----------------------------------------------------------------------
01112 void Node::messageRun() {
01113   (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
01114 }
01115 
01116 
01117 //-----------------------------------------------------------------------
01118 // run(void) runs the specified simulation for the specified number of
01119 // steps, overriding the contents of the configuration file
01120 //-----------------------------------------------------------------------
01121 void Node::run()
01122 {
01123   // Start Controller (aka scalar Sequencer) on Pe(0)
01124 //  printf("\n\n I am in Node.C in run method about to call  state->runController\n\n");
01125   if ( ! CkMyPe() ) {
01126     state->runController();
01127   }
01128 
01129   DebugM(4, "Starting Sequencers\n");
01130   // Run Sequencer on each HomePatch - i.e. start simulation
01131   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01132   ResizeArrayIter<HomePatchElem> ai(*hpl);
01133   for (ai=ai.begin(); ai != ai.end(); ai++) {
01134     HomePatch *patch = (*ai).patch;
01135 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
01136     patch->runSequencer();
01137   }
01138 
01139   if (!CkMyPe()) {
01140     double newTime = CmiWallTimer();
01141     iout << iINFO << "Startup phase " << startupPhase-1 << " took "
01142          << newTime - startupTime << " s, "
01143          << memusage_MB() << " MB of memory in use\n";
01144     iout << iINFO << "Finished startup at " << newTime << " s, "
01145          << memusage_MB() << " MB of memory in use\n\n" << endi;
01146     fflush(stdout);
01147   }
01148   
01149 }
01150 
01151 
01152 //-----------------------------------------------------------------------
01153 // Node scriptBarrier() - twiddle parameters with simulation halted
01154 //-----------------------------------------------------------------------
01155 
01156 void Node::enableScriptBarrier() {
01157   CkStartQD(CkIndex_Node::scriptBarrier((CkQdMsg*)0),&thishandle);
01158 }
01159 
01160 void Node::scriptBarrier(CkQdMsg *qmsg) {
01161   delete qmsg;
01162   //script->awaken();
01163 }
01164 
01165 void Node::scriptParam(ScriptParamMsg *msg) {
01166   simParameters->scriptSet(msg->param,msg->value);
01167   delete msg;
01168 }
01169 
01170 void Node::reloadCharges(const char *filename) {
01171   FILE *file = fopen(filename,"r");
01172   if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
01173 
01174   int n = molecule->numAtoms;
01175   float *charge = new float[n];
01176 
01177   for ( int i = 0; i < n; ++i ) {
01178     if ( ! fscanf(file,"%f",&charge[i]) )
01179       NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
01180   }
01181 
01182   fclose(file);
01183   CProxy_Node(thisgroup).reloadCharges(charge,n);
01184   delete [] charge;
01185 }
01186 
01187 void Node::reloadCharges(float charge[], int n) {
01188   molecule->reloadCharges(charge,n);
01189 }
01190 
01191 
01192 // BEGIN gf
01193 void Node::reloadGridforceGrid(const char * key) {
01194     DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
01195     
01196     int gridnum;
01197     MGridforceParams *mgridParams;
01198     if (key == NULL) {
01199         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01200         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01201     } else {
01202         gridnum = simParameters->mgridforcelist.index_for_key(key);
01203         mgridParams = simParameters->mgridforcelist.find_key(key);
01204     }
01205     
01206     if (gridnum < 0 || mgridParams == NULL) {
01207         NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
01208     }
01209     
01210     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01211     if (grid == NULL) {
01212         NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
01213     }
01214     grid->reinitialize(simParameters, mgridParams);
01215     
01216     CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
01217     
01218     DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
01219 }
01220 
01221 void Node::updateGridScale(char* key, Vector scale) {
01222     DebugM(4, "updateGridScale(char*, Vector) called on node " << CkMyPe() << "\n" << endi);
01223     
01224     int gridnum;
01225     MGridforceParams* mgridParams;
01226     if (key == NULL) {
01227         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01228         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01229     } else {
01230         gridnum = simParameters->mgridforcelist.index_for_key(key);
01231         mgridParams = simParameters->mgridforcelist.find_key(key);
01232     }
01233 
01234     if (gridnum < 0 || mgridParams == NULL) {
01235         NAMD_die("Node::updateGridScale(char*, Vector): Could not find grid.");
01236     }
01237     
01238     GridforceGrid* grid = molecule->get_gridfrc_grid(gridnum);
01239     if (grid == NULL) {
01240         NAMD_bug("Node::updateGridScale(char*, Vector): grid not found");
01241     }
01242     CProxy_Node(thisgroup).updateGridScale(gridnum, scale.x, scale.y, scale.z);
01243     
01244     DebugM(4, "updateGridScale(char*, Vector) finished\n" << endi);
01245 }
01246 void Node::updateGridScale(int gridnum, float sx, float sy, float sz) {
01247     if (CmiMyRank()) return;
01248     DebugM(4, "updateGridScale(char*, int, float, float, float) called on node " << CkMyPe() << "\n" << endi);
01249        
01250     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01251     if (grid == NULL) {
01252         NAMD_bug("Node::updateGridScale(char*, int, float, float, float):grid not found");
01253     }
01254     
01255     Vector scale(sx,sy,sz);
01256     simParameters->mgridforcelist.at_index(gridnum)->gridforceScale = scale;
01257     grid->set_scale( scale );
01258 
01259     DebugM(4, "updateGridScale(char*, int, float, float, float) finished\n" << endi);
01260 }
01261 
01262 void Node::reloadGridforceGrid(int gridnum) {
01263     if (CmiMyRank()) return;
01264     DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
01265     
01266     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01267     if (grid == NULL) {
01268         NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
01269     }
01270     
01271     if (CkMyPe()) {
01272         // not node 0 -> receive grid
01273         DebugM(4, "Receiving grid\n");
01274         
01275         delete grid;
01276         
01277         MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
01278         grid = GridforceGrid::unpack_grid(gridnum, msg);
01279         molecule->set_gridfrc_grid(gridnum, grid);
01280         delete msg;
01281     } else {
01282         // node 0 -> send grid
01283         DebugM(4, "Sending grid\n");
01284         
01285         MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
01286         GridforceGrid::pack_grid(grid, msg);
01287         msg->end();
01288         delete msg;
01289     }
01290     
01291     DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
01292 }
01293 // END gf
01294 
01295 
01296 // initiating replica
01297 void Node::sendCheckpointReq(int remote, const char *key, int task, Lattice &lat, ControllerState &cs) {
01298   CheckpointMsg *msg = new (1+strlen(key),0) CheckpointMsg;
01299   msg->replica = CmiMyPartition();
01300   msg->task = task;
01301   msg->checkpoint.lattice = lat;
01302   msg->checkpoint.state = cs;
01303   strcpy(msg->key,key);
01304   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01305   CmiSetHandler(env,recvCheckpointCReq_index);
01306 #if CMK_HAS_PARTITION
01307   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01308 #else
01309   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01310 #endif
01311 }
01312 
01313 // responding replica
01314 extern "C" {
01315   void recvCheckpointCReq_handler(envelope *env) {
01316     Node::Object()->recvCheckpointReq(CheckpointMsg::unpack(EnvToUsr(env)));
01317   }
01318 }
01319 
01320 // responding replica
01321 void Node::recvCheckpointReq(CheckpointMsg *msg) {
01322   state->controller->recvCheckpointReq(msg->key,msg->task,msg->checkpoint);
01323 
01324   int remote = msg->replica;
01325   msg->replica = CmiMyPartition();
01326   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01327   CmiSetHandler(env,recvCheckpointCAck_index);
01328 #if CMK_HAS_PARTITION
01329   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01330 #else
01331   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01332 #endif
01333 }
01334 
01335 // initiating replica
01336 extern "C" {
01337   void recvCheckpointCAck_handler(envelope *env) {
01338     Node::Object()->recvCheckpointAck(CheckpointMsg::unpack(EnvToUsr(env)));
01339   }
01340 }
01341 
01342 // initiating replica
01343 void Node::recvCheckpointAck(CheckpointMsg *msg) {
01344   state->controller->recvCheckpointAck(msg->checkpoint);
01345   delete msg;
01346 }
01347 
01348 
01349 void Node::sendEnableExitScheduler(void) {
01350   //CmiPrintf("sendEnableExitScheduler\n");
01351   CkQdMsg *msg = new CkQdMsg;
01352   CProxy_Node nodeProxy(thisgroup);
01353   nodeProxy[0].recvEnableExitScheduler(msg);
01354 }
01355 
01356 void Node::recvEnableExitScheduler(CkQdMsg *msg) {
01357   //CmiPrintf("recvEnableExitScheduler\n");
01358   delete msg;
01359   enableExitScheduler();
01360 }
01361 
01362 void Node::enableExitScheduler(void) {
01363   if ( CkMyPe() ) {
01364     sendEnableExitScheduler();
01365   } else {
01366     CkStartQD(CkIndex_Node::exitScheduler((CkQdMsg*)0),&thishandle);
01367   }
01368 }
01369 
01370 void Node::exitScheduler(CkQdMsg *msg) {
01371   //CmiPrintf("exitScheduler %d\n",CkMyPe());
01372   CsdExitScheduler();
01373   delete msg;
01374 }
01375 
01376 void Node::sendEnableEarlyExit(void) {
01377   CkQdMsg *msg = new CkQdMsg;
01378   CProxy_Node nodeProxy(thisgroup);
01379   nodeProxy[0].recvEnableEarlyExit(msg);
01380 }
01381 
01382 void Node::recvEnableEarlyExit(CkQdMsg *msg) {
01383   delete msg;
01384   enableEarlyExit();
01385 }
01386 
01387 void Node::enableEarlyExit(void) {
01388   if ( CkMyPe() ) {
01389     sendEnableEarlyExit();
01390   } else {
01391     CkStartQD(CkIndex_Node::earlyExit((CkQdMsg*)0),&thishandle);
01392   }
01393 }
01394 
01395 void Node::earlyExit(CkQdMsg *msg) {
01396   iout << iERROR << "Exiting prematurely; see error messages above.\n" << endi;
01397   if ( CmiNumPartitions() > 1 ) NAMD_quit("Exiting prematurely; see error messages above.");
01398   BackEnd::exit();
01399   delete msg;
01400 }
01401 
01402 
01403 //------------------------------------------------------------------------
01404 // Some odd utilities
01405 //------------------------------------------------------------------------
01406 void Node::saveMolDataPointers(NamdState *state)
01407 {
01408   this->molecule = state->molecule;
01409   this->parameters = state->parameters;
01410   this->simParameters = state->simParameters;
01411   this->configList = state->configList;
01412   this->pdb = state->pdb;
01413   this->state = state;
01414 }
01415 
01416 // entry methods for BG/P HPM (performance counters) library
01417 void Node::startHPM() {
01418 #if USE_HPM
01419   HPM_Start("500 steps", localRankOnNode);
01420 #endif
01421 }
01422 
01423 void Node::stopHPM() {
01424 #if USE_HPM
01425   HPM_Stop("500 steps", localRankOnNode);
01426   HPM_Print(CkMyPe(), localRankOnNode);
01427 #endif
01428 }
01429 
01430 void Node::traceBarrier(int turnOnTrace, int step){
01431         curTimeStep = step;
01432         if(turnOnTrace) traceBegin();
01433         else traceEnd();
01434 
01435     if(turnOnTrace) CmiTurnOnStats();
01436     else CmiTurnOffStats();
01437 
01438         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());    
01439         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01440         CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
01441         contribute(0, NULL, CkReduction::sum_int, cb);
01442         
01443 }
01444 
01445 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
01446         CmiAssert(CmiMyPe()==0);
01447         delete msg;     
01448         state->controller->resumeAfterTraceBarrier(curTimeStep);
01449 }
01450 
01451 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
01452 #ifdef MEASURE_NAMD_WITH_PAPI
01453         curMFlopStep = step;
01454         double results[NUM_PAPI_EVENTS+1];
01455 
01456         if(turnOnMeasure){              
01457           CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]=CmiWallTimer();
01458 
01459           long long counters[NUM_PAPI_EVENTS+1];
01460           int ret=PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
01461           if(ret==PAPI_OK)
01462             {
01463               //              CkPrintf("traceBarrier start counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01464             }
01465           else
01466             {
01467               CkPrintf("error PAPI_start_counters (%d) at step %d called on proc %d\n",ret , step, CkMyPe());
01468             }
01469           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)!=PAPI_OK)
01470             {
01471               CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01472             };
01473         }else{
01474           long long counters[NUM_PAPI_EVENTS+1];
01475           for(int i=0;i<NUM_PAPI_EVENTS;i++)  counters[i]=0LL;
01476           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)==PAPI_OK)
01477             {
01478 #if !MEASURE_PAPI_SPP
01479               results[0] = (double)counters[0]/1e6;
01480               results[1] = (double)counters[1]/1e6;
01481 #else
01482               for(int i=0;i<NUM_PAPI_EVENTS;i++)  results[i] = counters[i]/1e6;
01483 #endif
01484               //              for(int i=0;i<NUM_PAPI_EVENTS;i++) CkPrintf("[%d] counter %d is %ld\n",CkMyPe(),i,counters[i]);
01485             }
01486           else
01487             {
01488               //              CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01489             }
01490           //      CkPrintf("traceBarrier stop counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01491                 
01492           PAPI_stop_counters(counters, NUM_PAPI_EVENTS);        
01493         }
01494         if(CkMyPe()==0)
01495           //        CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01496         results[NUM_PAPI_EVENTS]=CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]; //starttime
01497         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01498         CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
01499         contribute(sizeof(double)*(NUM_PAPI_EVENTS+1), &results, CkReduction::sum_double, cb);  
01500 #endif
01501 }
01502 
01503 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
01504 #ifdef MEASURE_NAMD_WITH_PAPI
01505   
01506         if(simParameters->papiMeasureStartStep != curMFlopStep) {
01507                 double *results = (double *)msg->getData();
01508                 double endtime=CmiWallTimer();
01509                 int bstep = simParameters->papiMeasureStartStep;
01510                 int estep = bstep + simParameters->numPapiMeasureSteps;
01511 #if MEASURE_PAPI_SPP
01512                 CkPrintf("SPP INFO: PAPI_FP_OPS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[0]);
01513                 CkPrintf("SPP INFO: PAPI_TOT_INS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[1]);
01514                 CkPrintf("SPP INFO: perf::PERF_COUNT_HW_CACHE_LL:MISS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[2]);
01515                 CkPrintf("SPP INFO: DATA_PREFETCHER:ALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[3]);
01516                 CkPrintf("SPP INFO: PAPI_L1_DCA timestep %d to %d is %lf(1e6)\n", bstep,estep,results[4]);
01517                 CkPrintf("SPP INFO: PAPI_TOT_CYC timestep %d to % is %lf(1e6)\n", bstep,estep,results[5]);
01518                 //              CkPrintf("SPP INFO: INSTRUCTION_FETCH_STALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[6]);
01519                 //              CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]/CkNumPes());
01520                 CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]);
01521                 CkPrintf("SPP INFO: endtime %lf avgtime %lf tottime %lf\n", endtime,results[NUM_PAPI_EVENTS]/CkNumPes(),results[NUM_PAPI_EVENTS] );
01522 #else
01523                 if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
01524                         double totalFPIns = results[0];
01525                         if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
01526                         CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n", 
01527                                          bstep, estep, totalFPIns/CkNumPes());
01528                 }else{
01529                         char nameBuf[PAPI_MAX_STR_LEN];
01530                         CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ", 
01531                                          bstep, estep);
01532                         for(int i=0; i<NUM_PAPI_EVENTS; i++) {
01533                                 PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
01534                                 CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
01535                         }
01536                         CkPrintf("per processor\n");
01537                 }               
01538 #endif
01539         }
01540         delete msg;     
01541         state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
01542 #endif
01543 }
01544 
01545 extern char *gNAMDBinaryName;
01546 void Node::outputPatchComputeMaps(const char *filename, int tag){
01547         if(!simParameters->outputMaps && !simParameters->simulateInitialMapping) return;
01548 
01549         int numpes = CkNumPes();
01550         int nodesize = CkMyNodeSize();
01551         if(simParameters->simulateInitialMapping) {
01552                 numpes = simParameters->simulatedPEs;
01553                 nodesize = simParameters->simulatedNodeSize;
01554         }
01555 
01556         char fname[128];
01557         sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
01558 
01559         FILE *fp = fopen(fname, "w");
01560         if(fp == NULL) {
01561                 NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
01562                 return;
01563         }
01564         PatchMap *pMap = PatchMap::Object();
01565         ComputeMap *cMap = ComputeMap::Object();
01566         int numPatches = pMap->numPatches();
01567         int numComputes = cMap->numComputes();
01568         fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes, 
01569                         pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
01570         //output PatchMap info
01571         for(int i=0; i<numPatches; i++) {
01572         #ifdef MEM_OPT_VERSION
01573                 fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
01574         #else
01575                 fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
01576         #endif
01577         }
01578 
01579         //output ComputeMap info
01580         for(int i=0; i<numComputes; i++) {              
01581                 fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));             
01582         }
01583 }
01584 
01585 
01586 //======================================================================
01587 // Private functions
01588 
01589 #include "Node.def.h"
01590 

Generated on Wed Jan 17 01:17:14 2018 for NAMD by  doxygen 1.4.7