Node.C

Go to the documentation of this file.
00001 
00007 /*
00008    Toplevel routines for initializing a Node for a simulation
00009    one Node per Pe (processor element).
00010 */
00011 
00012 #if !defined(WIN32) || defined(__CYGWIN__)
00013 #include <unistd.h>
00014 #endif
00015 #include "InfoStream.h"
00016 #include "Node.decl.h"
00017 #include "Node.h"
00018 #ifdef DPMTA
00019 #include <pvm3.h>
00020 #endif
00021 
00022 #include "ProcessorPrivate.h"
00023 
00024 #define MIN_DEBUG_LEVEL 3
00025 //#define DEBUGM
00026 #include "Debug.h"
00027 
00028 #include <stdio.h>
00029 #include <converse.h>
00030 #include "memusage.h"
00031 #include "IMDOutput.h"
00032 #include "Lattice.h"
00033 #include "ComputeMsmMsa.h"  // needed for MsmMsaData definition
00034 #include "ComputeMsm.h"     // needed for MsmInitMsg definition
00035 #include "main.decl.h"
00036 #include "main.h"
00037 #include "WorkDistrib.h"
00038 #include "PatchMgr.h"
00039 #include "Patch.h"
00040 #include "Compute.h"
00041 #include "ComputeMap.h"
00042 #include "ComputeMgr.h"
00043 #include "Molecule.h"
00044 #include "HomePatchList.h"
00045 #include "AtomMap.h"
00046 #include "Sequencer.h"
00047 #include "Controller.h"
00048 #include "NamdState.h"
00049 #include "Output.h"
00050 #include "ProxyMgr.h"
00051 #include "PatchMap.h"
00052 #include "PatchMap.inl"
00053 #include "Parameters.h"
00054 #include "SimParameters.h"
00055 #include "Communicate.h"
00056 #include "LdbCoordinator.h"
00057 #include "ScriptTcl.h"
00058 #include "ComputeMgr.decl.h"
00059 #include "ComputePmeMgr.decl.h"
00060 // #ifdef NAMD_CUDA
00061 #include "ComputeCUDAMgr.decl.h"
00062 #include "ComputePmeCUDAMgr.decl.h"
00063 #include "ComputePmeCUDAMgr.h"
00064 // #endif
00065 #include "ComputeGridForceMgr.decl.h"
00066 #include "OptPmeMgr.decl.h"
00067 #include "Sync.h"
00068 #include "BackEnd.h"
00069 #include "PDB.h"
00070 #include "packmsg.h"
00071 #include "CollectionMgr.decl.h"
00072 #include "ParallelIOMgr.decl.h"
00073 #include "Vector.h"
00074 // BEGIN LA
00075 #include "Random.h"
00076 // END LA
00077 
00078 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00079 extern "C" void CApplicationInit();
00080 #endif
00081 
00082 #include "DumpBench.h"
00083 
00084 class CheckpointMsg : public CMessage_CheckpointMsg {
00085 public:
00086   int task;
00087   int replica;
00088   Controller::checkpoint checkpoint;
00089   char *key;
00090 };
00091 
00092 extern "C" {
00093   void recvCheckpointCReq_handler(envelope*);
00094   void recvCheckpointCAck_handler(envelope*);
00095 }
00096 
00097 #ifdef CMK_BALANCED_INJECTION_API
00098 #include "ckBIconfig.h"
00099 #endif
00100 
00101 #include "CollectionMgr.h"
00102 #include "CollectionMaster.h"
00103 #include "CollectionMgr.decl.h"
00104 #include "CollectionMaster.decl.h"
00105 
00106 #if USE_HPM
00107 extern "C" void HPM_Init(int);
00108 extern "C" void HPM_Start(char *label, int);
00109 extern "C" void HPM_Stop(char *label, int);
00110 extern "C" void HPM_Print(int, int);
00111 #endif
00112 
00113 #if defined(NAMD_MIC)
00114   extern void mic_dumpHostDeviceComputeMap();
00115   extern void mic_initHostDeviceLDB();
00116 #endif
00117 
00118 #ifdef MEASURE_NAMD_WITH_PAPI
00119 #include "papi.h"
00120 #if CMK_SMP
00121 #include <pthread.h>
00122 #endif
00123 #define NUM_PAPI_EVENTS 6
00124 CkpvDeclare(int *, papiEvents);
00125 
00126 #define MEASURE_PAPI_SPP 1
00127 #define MEASURE_PAPI_CACHE 0
00128 #define MEASURE_PAPI_FLOPS 0
00129 
00130 static void namdInitPapiCounters(){
00131         if(CkMyRank()==0){
00132                 //only initialize per OS process (i.e. a charm node)
00133                 int retval = PAPI_library_init(PAPI_VER_CURRENT);
00134                 if(retval != PAPI_VER_CURRENT) {
00135                         if(CkMyPe()==0){
00136                                 NAMD_die("PAPI library is not compatitible!");
00137                         }
00138                 }
00139         #if CMK_SMP
00140                 //now only consider systems that are compatible with POSIX
00141                 if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
00142                         if(CkMyPe()==0){
00143                                 NAMD_die("Multi-thread mode in PAPI could not be initialized!");
00144                         }
00145                 }
00146         #endif
00147         }
00148         CkpvInitialize(int *, papiEvents);
00149         CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS+1];
00150 
00151 #if MEASURE_PAPI_CACHE
00152         if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
00153                 CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
00154         }else{
00155                 if(CkMyPe()==0){
00156                         CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");                     
00157                 }
00158                 //if not default to PAPI_TOT_INS
00159                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00160         }
00161 
00162         if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
00163                 CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
00164         }else{
00165                 //if not default to PAPI_TOT_CYC
00166                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00167         }       
00168 #elif MEASURE_PAPI_FLOPS
00169         if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
00170                 CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
00171         }else{
00172                 if(CkMyPe()==0){
00173                         CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
00174                 }
00175                 //if not default to PAPI_TOT_INS
00176                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00177         }
00178 
00179         if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
00180                 CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
00181         }else{
00182                 //if not default to PAPI_TOT_CYC
00183                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00184         }
00185 #elif MEASURE_PAPI_SPP
00186 /* for SPP we record these
00187 1) PAPI_FP_OPS
00188 2) PAPI_TOT_INS
00189 3) perf::PERF_COUNT_HW_CACHE_LL:MISS
00190 4) DATA_PREFETCHER:ALL
00191 5) PAPI_L1_DCA
00192 6) INSTRUCTION_FETCH_STALL
00193 7) PAPI_TOT_CYC, and 
00194 8) real (wall) time
00195 */
00196         int papiEventSet = PAPI_NULL; 
00197         if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
00198           CmiAbort("PAPI failed to create event set!\n");
00199         }
00200 
00201         if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
00202                 CkpvAccess(papiEvents)[0] = PAPI_FP_OPS;
00203         }else{
00204                 if(CkMyPe()==0){
00205                         CkAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
00206                 }
00207         }
00208         if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
00209                 CkpvAccess(papiEvents)[1] = PAPI_TOT_INS;
00210         }else{
00211                 if(CkMyPe()==0){
00212                         CkAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
00213                 }
00214         }
00215         int EventCode;
00216         int ret;
00217         ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
00218         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00219           CkpvAccess(papiEvents)[2] = EventCode;
00220         }else{
00221                 if(CkMyPe()==0){
00222                         CkAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
00223                 }
00224         }
00225         ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
00226         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00227           CkpvAccess(papiEvents)[3] = EventCode;
00228         }else{
00229                 if(CkMyPe()==0){
00230                         CkAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
00231                 }
00232         }
00233         if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
00234                 CkpvAccess(papiEvents)[4] = PAPI_L1_DCA;
00235         }else{
00236                 if(CkMyPe()==0){
00237                         CkAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
00238                 }
00239         }
00240         /*      ret=PAPI_event_name_to_code("INSTRUCTION_FETCH_STALL",&EventCode);
00241         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00242           CkpvAccess(papiEvents)[5] = EventCode;
00243         }else{
00244                 if(CkMyPe()==0){
00245                         CkAbort("WARNING: INSTRUCTION_FETCH_STALL doesn't exist on this platform!");
00246                 }
00247         }
00248         */
00249         if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
00250                 CkpvAccess(papiEvents)[5] = PAPI_TOT_CYC;
00251         }else{
00252                 if(CkMyPe()==0){
00253                         CkAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
00254                 }
00255         }
00256         for(int i=0;i<NUM_PAPI_EVENTS;i++)
00257           {
00258             int papiRetValue=PAPI_add_events(papiEventSet, &CkpvAccess(papiEvents)[i],1);
00259             if (papiRetValue != PAPI_OK) {
00260               CkPrintf("failure for event %d\n",i);
00261               if (papiRetValue == PAPI_ECNFLCT) {
00262                 CmiAbort("PAPI events conflict! Please re-assign event types!\n");
00263               } else {
00264                 CmiAbort("PAPI failed to add designated events!\n");
00265               }
00266             }
00267             
00268           }
00269 #endif
00270 }
00271 #endif
00272 
00273 #ifdef OPENATOM_VERSION
00274 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
00275 {
00276   CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
00277 }
00278 #endif //OPENATOM_VERSION
00279 
00280 //======================================================================
00281 // Public Functions
00282 
00283 //----------------------------------------------------------------------
00284 
00285 int eventEndOfTimeStep;
00286 double startupTime;
00287 
00288 //----------------------------------------------------------------------
00289 // BOC constructor
00290 Node::Node(GroupInitMsg *msg)
00291 {    
00292   DebugM(4,"Creating Node\n");
00293 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00294   CApplicationInit();
00295 #endif
00296   if (CkpvAccess(Node_instance) == 0) {
00297     CkpvAccess(Node_instance) = this;
00298     eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep");
00299   } else {
00300     NAMD_bug("Node::Node() - another instance of Node exists!");
00301   }
00302 
00303   CkpvAccess(BOCclass_group) = msg->group;
00304   delete msg;
00305 
00306   CkpvAccess(BOCclass_group).node = thisgroup;
00307 
00308   recvCheckpointCReq_index = CmiRegisterHandler((CmiHandler)recvCheckpointCReq_handler);
00309   recvCheckpointCAck_index = CmiRegisterHandler((CmiHandler)recvCheckpointCAck_handler);
00310 
00311   startupPhase = 0;
00312 
00313   molecule = NULL;
00314   parameters = NULL;
00315   simParameters = NULL;
00316   configList = NULL;
00317   pdb = NULL;
00318   state = NULL;
00319   output = NULL;
00320   imd = new IMDOutput;
00321   colvars = 0;
00322 
00323 #if USE_HPM
00324   // assumes that this will be done only on BG/P
00325   TopoManager *tmgr = new TopoManager();
00326   int x, y, z;
00327   tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
00328   delete tmgr;
00329 #endif
00330 
00331   specialTracing = traceAvailable() && (traceIsOn()==0);
00332 
00333   DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
00334   patchMap = PatchMap::Instance();
00335   atomMap = AtomMap::Instance();
00336   if ( CkMyRank() == 0 ) ComputeMap::Instance();
00337 
00338   //Note: Binding BOC vars such as workDistrib has been moved
00339   //to the 1st phase of startup because the in-order message delivery
00340   //is not always guaranteed --Chao Mei
00341 #ifdef CMK_BALANCED_INJECTION_API
00342   if(CkMyRank() == 0){
00343     balancedInjectionLevel=ck_get_GNI_BIConfig();
00344     // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00345     ck_set_GNI_BIConfig(20);
00346     // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00347   }
00348 #endif
00349 
00350 }
00351 
00352 //----------------------------------------------------------------------
00353 // ~Node(void) needs to clean up everything.
00354 
00355 Node::~Node(void)
00356 {
00357   delete output;
00358   delete computeMap;
00359   delete atomMap;
00360   delete patchMap;
00361   delete CkpvAccess(comm);
00362   // BEGIN LA
00363   delete rand;
00364   // END LA
00365 #ifdef MEASURE_NAMD_WITH_PAPI
00366   delete CkpvAccess(papiEvents);
00367 #endif
00368 }
00369 
00370 void Node::bindBocVars(){
00371     DebugM(4,"Binding to BOC's\n");
00372     CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00373     patchMgr = pm.ckLocalBranch();
00374     CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
00375     proxyMgr = prm.ckLocalBranch();
00376     CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
00377     workDistrib = wd.ckLocalBranch();
00378     CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00379     computeMgr = cm.ckLocalBranch();
00380     CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
00381     ldbCoordinator = lc.ckLocalBranch();
00382   #ifdef MEM_OPT_VERSION      
00383     CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
00384     ioMgr = io.ckLocalBranch();
00385   #endif
00386 
00387 }
00388 
00389 //----------------------------------------------------------------------
00390 // Malloc Test Sequence
00391 void Node::mallocTest(int step) {
00392   int MB = 1024*1024;
00393   int size = 100;
00394   char* foo = (char*) malloc(size*MB);
00395   if ( ! foo ) {
00396     char buf[256];
00397     sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
00398     NAMD_die(buf);
00399   }
00400   memset(foo,0,size*MB*sizeof(char));
00401 }
00402 
00403 void Node::mallocTestQd(CkQdMsg *qmsg) {
00404   delete qmsg;
00405   if ( mallocTest_size ) {
00406     CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
00407   } else {
00408     CkPrintf("Starting malloc test on all PEs.\n");
00409   }
00410   fflush(stdout);
00411   ++mallocTest_size;
00412   CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00413   (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
00414 }
00415 
00416 //----------------------------------------------------------------------
00417 // Startup Sequence
00418 
00419 void Node::messageStartUp() {
00420   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00421 }
00422 
00423 void Node::startUp(CkQdMsg *qmsg) {
00424   delete qmsg;
00425   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00426 }
00427 
00428 SimParameters *node_simParameters;
00429 Parameters *node_parameters;
00430 Molecule *node_molecule;
00431 
00432 extern void registerUserEventsForAllComputeObjs(void);
00433 
00434 void Node::startup() {
00435   int gotoRun = false;
00436   double newTime;
00437 
00438   if (!CkMyPe()) {
00439     if (!startupPhase) {
00440       iout << iINFO << "\n";
00441       startupTime = CmiWallTimer();
00442       iout << iINFO << "Entering startup at " << startupTime << " s, ";
00443     } else {
00444       newTime = CmiWallTimer();
00445       iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00446            << newTime - startupTime << " s, ";
00447       startupTime = newTime;
00448     }
00449     iout << memusage_MB() << " MB of memory in use\n" << endi;
00450     fflush(stdout);
00451   }
00452   switch (startupPhase) {
00453 
00454   case 0:
00455     computeMap = ComputeMap::Object();
00456     namdOneCommInit(); // Namd1.X style
00457   break;
00458 
00459   case 1:
00460       bindBocVars();
00461 
00462     // send & receive molecule, simparameters... (Namd1.X style)
00463     if (CkMyPe()) {
00464       namdOneRecv();
00465     } else {
00466       namdOneSend();
00467     }
00468   break;
00469 
00470   case 2:
00471     // fix up one-per-node objects (for SMP version)
00472     simParameters = node_simParameters;
00473     parameters = node_parameters;
00474     molecule = node_molecule;
00475 
00476     SimParameters::nonbonded_select();
00477     
00478     #if !CMK_SMP || ! USE_CKLOOP
00479     //the CkLoop library should be only used in SMP mode
00480     simParameters->useCkLoop = 0;
00481     #else
00482     if ( CkNumPes() < 2 * CkNumNodes() ) simParameters->useCkLoop = 0;
00483     #endif
00484 
00485 
00486     if ( simParameters->mallocTest ) {
00487       if (!CkMyPe()) {
00488         mallocTest_size = 0;
00489         CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00490       }
00491       return;
00492     }
00493 
00494       
00495         #ifdef MEASURE_NAMD_WITH_PAPI
00496         if(simParameters->papiMeasure) namdInitPapiCounters();  
00497         #endif
00498     
00499     #ifdef MEM_OPT_VERSION
00500     //At this point, each Node object has received the simParameters,
00501     //parameters and the atom signatures info from the master Node
00502     //(proc 0). It's time to initialize the parallel IO manager and
00503     //read the binary per-atom file --Chao Mei
00504 
00505     //Step 1: initialize the parallel IO manager per Node
00506     ioMgr->initialize(this);
00507     #endif
00508 
00509   break;
00510 
00511   case 3:
00512 
00513     #ifdef MEM_OPT_VERSION
00514     //Step 2: read the binary per-atom files (signater index, coordinates etc.)
00515     ioMgr->readPerAtomInfo();
00516     #endif
00517 
00518   break;
00519 
00520   case 4:
00521 
00522     #ifdef MEM_OPT_VERSION
00523     //Step 3: update counters of tuples and exclusions inside Molecule object
00524     ioMgr->updateMolInfo();
00525 
00526     //Step 4: prepare distributing the atoms to neighboring procs if necessary
00527     ioMgr->migrateAtomsMGrp();
00528 
00529     //step 5: initialize patchMap and send it to every other processors
00530     //to decide atoms to patch distribution on every input processor
00531     if(!CkMyPe()) {
00532         workDistrib->patchMapInit(); // create space division
00533         workDistrib->sendPatchMap();
00534     }
00535     #endif
00536 
00537     #if USE_HPM
00538     HPM_Init(localRankOnNode);
00539     #endif    
00540 
00541     // take care of inital thread setting
00542     threadInit();
00543 
00544     // create blank AtomMap
00545     AtomMap::Object()->allocateMap(molecule->numAtoms);
00546 
00547     if (!CkMyPe()) {
00548       if (simParameters->useOptPME)
00549         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_OptPmeMgr::ckNew();
00550       else 
00551 #ifdef NAMD_CUDA
00552       if (simParameters->usePMECUDA) {
00553         // computePmeCUDAMgr was created in BackEnd.C
00554         // This empty branch is to avoid initializing ComputePmeMgr
00555       } else
00556 #endif
00557       if (simParameters->PMEOn) {
00558         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
00559       }
00560         #ifdef OPENATOM_VERSION
00561         if ( simParameters->openatomOn ) { 
00562           CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
00563         }
00564         #endif // OPENATOM_VERSION
00565 
00566     }
00567     
00568     #ifdef OPENATOM_VERSION
00569     if ( simParameters->openatomOn ) {
00570       // if ( ! CkMyPe() ) { 
00571         CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle); 
00572         startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
00573       // }
00574     }
00575     #endif // OPENATOM_VERSION
00576   
00577     // BEGIN LA
00578     rand = new Random(simParameters->randomSeed);
00579     rand->split(CkMyPe(), CkNumPes());
00580     // END LA
00581 
00582   break;
00583 
00584   case 5:
00585     #ifdef MEM_OPT_VERSION
00586     //Now, every input proc has received all the atoms necessary
00587     //to decide the patches those atoms belong to
00588     
00589     //step 1: integrate the migrated atoms into the atom list that
00590     //contains the initally distributed atoms, and sort the atoms
00591     //based on hydrogenList value
00592     ioMgr->integrateMigratedAtoms();
00593 
00594     //step 2: integrate the cluster size of each atom on each output proc
00595     ioMgr->integrateClusterSize();
00596 
00597     //step 3: calculate the number of atoms in each patch on every
00598     //input procs (atoms belonging to a patch may lie on different
00599     //procs), and reduce such info on proc 0. Such info is required
00600     //for determing which node a particular patch is assigned to.
00601     ioMgr->calcAtomsInEachPatch();
00602 
00603     //set to false to re-send PatchMap later
00604     workDistrib->setPatchMapArrived(false);
00605     #endif
00606     break;
00607   case 6:     
00608     if(simParameters->isSendSpanningTreeOn()) {                         
00609                         ProxyMgr::Object()->setSendSpanning();
00610     }
00611     if(simParameters->isRecvSpanningTreeOn()) {                         
00612                         ProxyMgr::Object()->setRecvSpanning();
00613     }
00614     if(simParameters->proxyTreeBranchFactor) {
00615                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);
00616     }
00617     #ifdef PROCTRACE_DEBUG
00618     DebugFileTrace::Instance("procTrace");
00619     #endif
00620 
00621     if (!CkMyPe()) {
00622       output = new Output; // create output object just on PE(0)
00623 
00624       #ifndef MEM_OPT_VERSION
00625       workDistrib->patchMapInit(); // create space division
00626       workDistrib->createHomePatches(); // load atoms into HomePatch(es)
00627       #endif
00628       
00629       workDistrib->assignNodeToPatch();
00630       workDistrib->mapComputes();         
00631       //ComputeMap::Object()->printComputeMap();
00632 
00633       // For MIC runs, take the additional step after the compute map has been created to
00634       //   assign the various computes to either the host or the device.  This info will
00635       //   be distributed across the PEs.
00636       #if defined(NAMD_MIC)
00637         mic_initHostDeviceLDB();
00638       #endif
00639           
00640           if(simParameters->simulateInitialMapping) {
00641           iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
00642                           << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
00643                   outputPatchComputeMaps("init_mapping", 0);
00644                   iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
00645                   BackEnd::exit();
00646           }
00647 
00648       registerUserEventsForAllComputeObjs();
00649 
00650       //in MEM_OPT_VERSION, patchMap is resent
00651       //because they have been updated since creation including
00652       //#atoms per patch, the proc a patch should stay etc. --Chao Mei
00653       workDistrib->sendPatchMap();
00654       #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00655       CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00656       //a node broadcast
00657       npm.createProxyInfo(PatchMap::Object()->numPatches());
00658       #endif
00659     }
00660     {
00661         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00662         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00663         if(CkMyRank()==0) {
00664             //just need to register once
00665             npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
00666         }
00667         npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
00668         #endif
00669     }
00670   break;
00671 
00672   case 7:
00673 #ifdef CHARM_HAS_MSA
00674     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00675       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00676       msm[CkMyPe()].initialize(new CkQdMsg);
00677     }
00678 #else
00679     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00680       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00681       MsmInitMsg *msg = new MsmInitMsg;
00682       Lattice lattice = simParameters->lattice;  // system lattice vectors
00683       ScaledPosition smin=0, smax=0;
00684       if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
00685         msg->smin = smin;
00686         msg->smax = smax;
00687         msm[CkMyPe()].initialize(msg);  // call from my own PE
00688       }
00689       else if ( ! CkMyPe() ) {
00690         pdb->get_extremes(smin, smax);  // only available on PE 0
00691         msg->smin = smin;
00692         msg->smax = smax;
00693         msm.initialize(msg);  // broadcast to chare group
00694       }
00695 
00696       /*
00697       CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00698       Node *node = nd.ckLocalBranch();
00699       ScaledPosition smin, smax;
00700       node->pdb->get_extremes(smin, smax);
00701       msg->smin = smin;                       // extreme positions in system
00702       msg->smax = smax;
00703       msm[CkMyPe()].initialize(msg);
00704       */
00705     }
00706 #endif
00707 
00708     if ( simParameters->PMEOn ) {
00709       if ( simParameters->useOptPME ) {
00710         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00711         pme[CkMyPe()].initialize(new CkQdMsg);
00712       }
00713       else {
00714         #ifdef OPENATOM_VERSION
00715         if ( simParameters->openatomOn ) { 
00716           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00717           moa[CkMyPe()].initialize(new CkQdMsg);
00718         }
00719         #endif // OPENATOM_VERSION
00720 #ifdef NAMD_CUDA
00721         if ( simParameters->usePMECUDA ) {
00722           if(CkMyRank()==0) {
00723             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00724             pme.ckLocalBranch()->initialize(new CkQdMsg);  // must run on pe 0 to call ckNew
00725           }
00726         } else 
00727 #endif
00728         {
00729           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00730           pme[CkMyPe()].initialize(new CkQdMsg);          
00731         }
00732       }
00733     }
00734 #ifdef NAMD_CUDA
00735     if ( (simParameters->useCUDA2 || simParameters->bondedCUDA) && CkMyRank()==0 ) {
00736       CProxy_ComputeCUDAMgr nb(CkpvAccess(BOCclass_group).computeCUDAMgr);
00737       nb[CkMyNode()].initialize(new CkQdMsg);
00738     }
00739 #endif
00740 
00741     if (!CkMyPe()) {
00742       workDistrib->sendComputeMap();
00743     }
00744 
00745     #ifdef MEM_OPT_VERSION
00746     //migrate atoms to HomePatch processors
00747     ioMgr->sendAtomsToHomePatchProcs();
00748     #endif
00749     break;
00750     
00751   case 8:
00752     // part 2 of MSM init
00753     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00754       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00755       msm[CkMyPe()].initialize_create();  // call from my own PE
00756     }
00757 
00758     if ( simParameters->PMEOn ) {
00759       if ( simParameters->useOptPME ) {
00760         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00761         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00762       }
00763       else {
00764         #ifdef OPENATOM_VERSION
00765         if ( simParameters->openatomOn ) { 
00766           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00767           moa[CkMyPe()].initWorkers(new CkQdMsg);
00768         }
00769         #endif // OPENATOM_VERSION
00770 #ifdef NAMD_CUDA
00771         if ( simParameters->usePMECUDA ) {
00772           if(CkMyRank()==0) {
00773             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00774             pme[CkMyNode()].initialize_pencils(new CkQdMsg);
00775           }
00776         } else
00777 #endif
00778         {
00779           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00780           pme[CkMyPe()].initialize_pencils(new CkQdMsg);          
00781         }
00782       }
00783     }
00784 #ifdef CHARM_HAS_MSA
00785     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00786       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00787       msm[CkMyPe()].initWorkers(new CkQdMsg);
00788     }
00789 #else
00790     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00791       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00792       msm[CkMyPe()].update(new CkQdMsg);
00793     }
00794 #endif
00795 
00796     #ifdef MEM_OPT_VERSION
00797     //Now every processor has all the atoms it needs to create the HomePatches.
00798     //The HomePatches are created in parallel on every home patch procs.
00799     ioMgr->createHomePatches();
00800     #else
00801     if (!CkMyPe()) {
00802       workDistrib->distributeHomePatches();          
00803     }
00804     #endif
00805   break;
00806 
00807   case 9:
00808     if ( simParameters->PMEOn ) {
00809       if ( simParameters->useOptPME ) {
00810         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00811         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00812       }
00813       else {
00814         #ifdef OPENATOM_VERSION
00815         if ( simParameters->openatomOn ) { 
00816           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00817           moa[CkMyPe()].startWorkers(new CkQdMsg);
00818         }
00819         #endif // OPENATOM_VERSION
00820 #ifdef NAMD_CUDA
00821         if ( simParameters->usePMECUDA ) {
00822           if(CkMyRank()==0) {
00823             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00824             pme[CkMyNode()].activate_pencils(new CkQdMsg);
00825           }
00826         } else
00827 #endif
00828         {
00829           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00830           pme[CkMyPe()].activate_pencils(new CkQdMsg);          
00831         }
00832       }
00833     }
00834 #ifdef CHARM_HAS_MSA
00835     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00836       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00837       msm[CkMyPe()].startWorkers(new CkQdMsg);
00838     }
00839 #else
00840     /*
00841     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00842       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00843       //msm[CkMyPe()].startWorkers(new CkQdMsg);
00844     }
00845     */
00846 #endif
00847 
00848     proxyMgr->createProxies();  // need Home patches before this
00849     if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
00850 
00851 #ifdef NAMD_TCL
00852     // TclInitSubsystems() has a race condition so we create one interp per node here
00853     if (CkMyPe() && CkMyNodeSize() > 1 && ! CkMyRank()) Tcl_DeleteInterp(Tcl_CreateInterp());
00854 #endif
00855 
00856 #ifdef USE_NODEPATCHMGR
00857         //at this point, PatchMap info has been recved on PEs. It is time to create
00858         //the home patch spanning tree for receiving proxy list info
00859         if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
00860                 if(CkMyRank()==0) {
00861                         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00862                         npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
00863                 }
00864         }
00865 #endif
00866 
00867   break;
00868 
00869   case 10:
00870 
00871     // DMK - DEBUG - If, in MIC runs, the debug option to dump all the compute maps to files
00872     //   for debugging/verification purposes has been enabled, have each PE do so now.
00873     #if defined(NAMD_MIC)
00874       mic_dumpHostDeviceComputeMap();
00875     #endif
00876 
00877     if (!CkMyPe()) {
00878       iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
00879            << " COMPUTE OBJECTS\n" << endi;
00880     }
00881     DebugM(4,"Creating Computes\n");
00882     computeMgr->createComputes(ComputeMap::Object());
00883     DebugM(4,"Building Sequencers\n");
00884     buildSequencers();
00885     DebugM(4,"Initializing LDB\n");
00886     LdbCoordinator::Object()->initialize(PatchMap::Object(),ComputeMap::Object());
00887   break;
00888 
00889   case 11:
00890     // computes may create proxies on the fly so put these in separate phase
00891     Sync::Object()->openSync();  // decide if to open local Sync 
00892     if (proxySendSpanning || proxyRecvSpanning ) proxyMgr->buildProxySpanningTree();
00893 #ifdef CMK_BALANCED_INJECTION_API
00894     if(CkMyRank() == 0){
00895       // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00896       ck_set_GNI_BIConfig(balancedInjectionLevel);
00897       // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00898     }
00899 #endif
00900 
00901   break;
00902 
00903   case 12:
00904     {
00905         //For debugging
00906         /*if(!CkMyPe()){
00907         FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
00908         dumpbench(dumpFile);
00909         NAMD_die("Normal execution\n");
00910         }*/
00911     }
00912     #ifdef MEM_OPT_VERSION
00913     //free space in the Molecule object that are not used anymore
00914     ioMgr->freeMolSpace();
00915     #endif
00916     gotoRun = true;
00917   break;
00918 
00919   default:
00920     NAMD_bug("Startup Phase has a bug - check case statement");
00921   break;
00922 
00923   }
00924 
00925   startupPhase++;
00926   if (!CkMyPe()) {
00927     if (!gotoRun) {
00928       CkStartQD(CkIndex_Node::startUp((CkQdMsg*)0),&thishandle);
00929     } else {
00930       Node::messageRun();
00931     }
00932   }
00933 }
00934 
00935 #ifdef OPENATOM_VERSION
00936 void Node::doneMoaStart()
00937 {
00938 #ifdef OPENATOM_VERSION_DEBUG
00939   CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
00940 #endif //OPENATOM_VERSION_DEBUG
00941 }
00942 #endif //OPENATOM_VERSION
00943 
00944 void Node::namdOneCommInit()
00945 {
00946   if (CkpvAccess(comm) == NULL) {
00947     CkpvAccess(comm) = new Communicate();
00948 #ifdef DPMTA
00949     pvmc_init();
00950 #endif
00951   }
00952 }
00953 
00954 // Namd 1.X style Send/Recv of simulation information
00955 
00956 void Node::namdOneRecv() {
00957   if ( CmiMyRank() ) return;
00958 
00959   MIStream *conv_msg;
00960 
00961   // Receive molecule and simulation parameter information
00962   simParameters = node_simParameters = new SimParameters;
00963   //****** BEGIN CHARMM/XPLOR type changes
00964   parameters = node_parameters = new Parameters();
00965   //****** END CHARMM/XPLOR type changes
00966   molecule = node_molecule = new Molecule(simParameters,parameters);
00967 
00968   DebugM(4, "Getting SimParameters\n");
00969   conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
00970   simParameters->receive_SimParameters(conv_msg);
00971 
00972   DebugM(4, "Getting Parameters\n");
00973   conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
00974   parameters->receive_Parameters(conv_msg);
00975 
00976   DebugM(4, "Getting Molecule\n");
00977   conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00978   // Modified by JLai -- 10.21.11
00979   molecule->receive_Molecule(conv_msg);
00980   if(simParameters->goForcesOn) {
00981     iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
00982     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00983     molecule->receive_GoMolecule(conv_msg);
00984   } 
00985   // End of modification
00986   DebugM(4, "Done Receiving\n");
00987 }
00988 
00989 void Node::namdOneSend() {
00990   node_simParameters = simParameters;
00991   node_parameters = parameters;
00992   node_molecule = molecule;
00993 
00994   MOStream *conv_msg;
00995   // I'm Pe(0) so I send what I know
00996   DebugM(4, "Sending SimParameters\n");  
00997   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
00998   simParameters->send_SimParameters(conv_msg);
00999 
01000   DebugM(4, "Sending Parameters\n");
01001   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
01002   parameters->send_Parameters(conv_msg);
01003 
01004   DebugM(4, "Sending Molecule\n");
01005   int bufSize = BUFSIZE;
01006   if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01007   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01008   // Modified by JLai -- 10.21.11
01009   molecule->send_Molecule(conv_msg);
01010   
01011   if(simParameters->goForcesOn) {
01012     iout << iINFO <<  "Master Node sending GoMolecule Information" << "\n" << endi;
01013     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01014     molecule->send_GoMolecule(conv_msg);
01015   } // End of modification
01016 }
01017 
01018 
01019 void Node::reloadStructure(const char *fname, const char *pdbname) {
01020   delete molecule;
01021   molecule = state->molecule = 0;
01022   delete pdb;
01023   pdb = state->pdb = 0;
01024   state->loadStructure(fname,pdbname,1);
01025   this->molecule = state->molecule;
01026   this->pdb = state->pdb;
01027   CProxy_Node nodeProxy(thisgroup);
01028   nodeProxy.resendMolecule();
01029 }
01030 
01031 
01032 void Node::resendMolecule() {
01033   if ( CmiMyRank() ) {
01034     return;
01035   }
01036   if ( CmiMyPe() == 0 ) {
01037     int bufSize = BUFSIZE;
01038     MOStream *conv_msg;
01039     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, bufSize);
01040     parameters->send_Parameters(conv_msg);
01041     if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01042     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01043     molecule->send_Molecule(conv_msg);
01044   } else {
01045     MIStream *conv_msg;
01046     delete parameters;
01047     parameters = new Parameters;
01048     conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
01049     parameters->receive_Parameters(conv_msg);
01050     delete molecule;
01051     molecule = new Molecule(simParameters,parameters);
01052     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
01053     molecule->receive_Molecule(conv_msg);
01054   }
01055   node_parameters = parameters;
01056   node_molecule = molecule;
01057   SimParameters::nonbonded_select();
01058   computeMgr->sendBuildCudaExclusions();
01059   CProxy_Node nodeProxy(thisgroup);
01060   for ( int i=0; i<CmiMyNodeSize(); ++i ) {
01061     nodeProxy[CmiMyPe()+i].resendMolecule2();
01062   }
01063 }
01064 
01065 void Node::resendMolecule2() {
01066   parameters = node_parameters;
01067   molecule = node_molecule;
01068   AtomMap::Object()->allocateMap(molecule->numAtoms);
01069 }
01070 
01071 
01072 // Initial thread setup
01073 
01074 void Node::threadInit() {
01075   // Thread initialization
01076   if (CthImplemented()) {
01077     CthSetStrategyDefault(CthSelf());
01078   } else {
01079     NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
01080   }
01081 }
01082 
01083 //
01084 void Node::buildSequencers() {
01085   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01086   ResizeArrayIter<HomePatchElem> ai(*hpl);
01087 
01088   // Controller object is only on Pe(0)
01089   if ( ! CkMyPe() ) {
01090     Controller *controller = new Controller(state);
01091     state->useController(controller);
01092   }
01093 
01094   // Assign Sequencer to all HomePatch(es)
01095   for (ai=ai.begin(); ai != ai.end(); ai++) {
01096     HomePatch *patch = (*ai).patch;
01097     Sequencer *sequencer = new Sequencer(patch);
01098     patch->useSequencer(sequencer);
01099   }
01100 }
01101 
01102 
01103 
01104 //-----------------------------------------------------------------------
01105 // Node run() - broadcast to all nodes
01106 //-----------------------------------------------------------------------
01107 void Node::messageRun() {
01108   (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
01109 }
01110 
01111 
01112 //-----------------------------------------------------------------------
01113 // run(void) runs the specified simulation for the specified number of
01114 // steps, overriding the contents of the configuration file
01115 //-----------------------------------------------------------------------
01116 void Node::run()
01117 {
01118   // Start Controller (aka scalar Sequencer) on Pe(0)
01119 //  printf("\n\n I am in Node.C in run method about to call  state->runController\n\n");
01120   if ( ! CkMyPe() ) {
01121     state->runController();
01122   }
01123 
01124   DebugM(4, "Starting Sequencers\n");
01125   // Run Sequencer on each HomePatch - i.e. start simulation
01126   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01127   ResizeArrayIter<HomePatchElem> ai(*hpl);
01128   for (ai=ai.begin(); ai != ai.end(); ai++) {
01129     HomePatch *patch = (*ai).patch;
01130 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
01131     patch->runSequencer();
01132   }
01133 
01134   if (!CkMyPe()) {
01135     double newTime = CmiWallTimer();
01136     iout << iINFO << "Startup phase " << startupPhase-1 << " took "
01137          << newTime - startupTime << " s, "
01138          << memusage_MB() << " MB of memory in use\n";
01139     iout << iINFO << "Finished startup at " << newTime << " s, "
01140          << memusage_MB() << " MB of memory in use\n\n" << endi;
01141     fflush(stdout);
01142   }
01143   
01144 }
01145 
01146 
01147 //-----------------------------------------------------------------------
01148 // Node scriptBarrier() - twiddle parameters with simulation halted
01149 //-----------------------------------------------------------------------
01150 
01151 void Node::enableScriptBarrier() {
01152   CkStartQD(CkIndex_Node::scriptBarrier((CkQdMsg*)0),&thishandle);
01153 }
01154 
01155 void Node::scriptBarrier(CkQdMsg *qmsg) {
01156   delete qmsg;
01157   //script->awaken();
01158 }
01159 
01160 void Node::scriptParam(ScriptParamMsg *msg) {
01161   simParameters->scriptSet(msg->param,msg->value);
01162   delete msg;
01163 }
01164 
01165 void Node::reloadCharges(const char *filename) {
01166   FILE *file = fopen(filename,"r");
01167   if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
01168 
01169   int n = molecule->numAtoms;
01170   float *charge = new float[n];
01171 
01172   for ( int i = 0; i < n; ++i ) {
01173     if ( ! fscanf(file,"%f",&charge[i]) )
01174       NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
01175   }
01176 
01177   fclose(file);
01178   CProxy_Node(thisgroup).reloadCharges(charge,n);
01179   delete [] charge;
01180 }
01181 
01182 void Node::reloadCharges(float charge[], int n) {
01183   molecule->reloadCharges(charge,n);
01184 }
01185 
01186 
01187 // BEGIN gf
01188 void Node::reloadGridforceGrid(const char * key) {
01189     DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
01190     
01191     int gridnum;
01192     MGridforceParams *mgridParams;
01193     if (key == NULL) {
01194         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01195         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01196     } else {
01197         gridnum = simParameters->mgridforcelist.index_for_key(key);
01198         mgridParams = simParameters->mgridforcelist.find_key(key);
01199     }
01200     
01201     if (gridnum < 0 || mgridParams == NULL) {
01202         NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
01203     }
01204     
01205     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01206     if (grid == NULL) {
01207         NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
01208     }
01209     grid->reinitialize(simParameters, mgridParams);
01210     
01211     CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
01212     
01213     DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
01214 }
01215 
01216 void Node::updateGridScale(char* key, Vector scale) {
01217     DebugM(4, "updateGridScale(char*, Vector) called on node " << CkMyPe() << "\n" << endi);
01218     
01219     int gridnum;
01220     MGridforceParams* mgridParams;
01221     if (key == NULL) {
01222         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01223         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01224     } else {
01225         gridnum = simParameters->mgridforcelist.index_for_key(key);
01226         mgridParams = simParameters->mgridforcelist.find_key(key);
01227     }
01228 
01229     if (gridnum < 0 || mgridParams == NULL) {
01230         NAMD_die("Node::updateGridScale(char*, Vector): Could not find grid.");
01231     }
01232     
01233     GridforceGrid* grid = molecule->get_gridfrc_grid(gridnum);
01234     if (grid == NULL) {
01235         NAMD_bug("Node::updateGridScale(char*, Vector): grid not found");
01236     }
01237     CProxy_Node(thisgroup).updateGridScale(gridnum, scale.x, scale.y, scale.z);
01238     
01239     DebugM(4, "updateGridScale(char*, Vector) finished\n" << endi);
01240 }
01241 void Node::updateGridScale(int gridnum, float sx, float sy, float sz) {
01242     if (CmiMyRank()) return;
01243     DebugM(4, "updateGridScale(char*, int, float, float, float) called on node " << CkMyPe() << "\n" << endi);
01244        
01245     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01246     if (grid == NULL) {
01247         NAMD_bug("Node::updateGridScale(char*, int, float, float, float):grid not found");
01248     }
01249     
01250     Vector scale(sx,sy,sz);
01251     simParameters->mgridforcelist.at_index(gridnum)->gridforceScale = scale;
01252     grid->set_scale( scale );
01253 
01254     DebugM(4, "updateGridScale(char*, int, float, float, float) finished\n" << endi);
01255 }
01256 
01257 void Node::reloadGridforceGrid(int gridnum) {
01258     if (CmiMyRank()) return;
01259     DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
01260     
01261     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01262     if (grid == NULL) {
01263         NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
01264     }
01265     
01266     if (CkMyPe()) {
01267         // not node 0 -> receive grid
01268         DebugM(4, "Receiving grid\n");
01269         
01270         delete grid;
01271         
01272         MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
01273         grid = GridforceGrid::unpack_grid(gridnum, msg);
01274         molecule->set_gridfrc_grid(gridnum, grid);
01275         delete msg;
01276     } else {
01277         // node 0 -> send grid
01278         DebugM(4, "Sending grid\n");
01279         
01280         MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
01281         GridforceGrid::pack_grid(grid, msg);
01282         msg->end();
01283         delete msg;
01284     }
01285     
01286     DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
01287 }
01288 // END gf
01289 
01290 
01291 // initiating replica
01292 void Node::sendCheckpointReq(int remote, const char *key, int task, Lattice &lat, ControllerState &cs) {
01293   CheckpointMsg *msg = new (1+strlen(key),0) CheckpointMsg;
01294   msg->replica = CmiMyPartition();
01295   msg->task = task;
01296   msg->checkpoint.lattice = lat;
01297   msg->checkpoint.state = cs;
01298   strcpy(msg->key,key);
01299   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01300   CmiSetHandler(env,recvCheckpointCReq_index);
01301 #if CMK_HAS_PARTITION
01302   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01303 #else
01304   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01305 #endif
01306 }
01307 
01308 // responding replica
01309 extern "C" {
01310   void recvCheckpointCReq_handler(envelope *env) {
01311     Node::Object()->recvCheckpointReq(CheckpointMsg::unpack(EnvToUsr(env)));
01312   }
01313 }
01314 
01315 // responding replica
01316 void Node::recvCheckpointReq(CheckpointMsg *msg) {
01317   state->controller->recvCheckpointReq(msg->key,msg->task,msg->checkpoint);
01318 
01319   int remote = msg->replica;
01320   msg->replica = CmiMyPartition();
01321   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01322   CmiSetHandler(env,recvCheckpointCAck_index);
01323 #if CMK_HAS_PARTITION
01324   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01325 #else
01326   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01327 #endif
01328 }
01329 
01330 // initiating replica
01331 extern "C" {
01332   void recvCheckpointCAck_handler(envelope *env) {
01333     Node::Object()->recvCheckpointAck(CheckpointMsg::unpack(EnvToUsr(env)));
01334   }
01335 }
01336 
01337 // initiating replica
01338 void Node::recvCheckpointAck(CheckpointMsg *msg) {
01339   state->controller->recvCheckpointAck(msg->checkpoint);
01340   delete msg;
01341 }
01342 
01343 
01344 void Node::sendEnableExitScheduler(void) {
01345   //CmiPrintf("sendEnableExitScheduler\n");
01346   CkQdMsg *msg = new CkQdMsg;
01347   CProxy_Node nodeProxy(thisgroup);
01348   nodeProxy[0].recvEnableExitScheduler(msg);
01349 }
01350 
01351 void Node::recvEnableExitScheduler(CkQdMsg *msg) {
01352   //CmiPrintf("recvEnableExitScheduler\n");
01353   delete msg;
01354   enableExitScheduler();
01355 }
01356 
01357 void Node::enableExitScheduler(void) {
01358   if ( CkMyPe() ) {
01359     sendEnableExitScheduler();
01360   } else {
01361     CkStartQD(CkIndex_Node::exitScheduler((CkQdMsg*)0),&thishandle);
01362   }
01363 }
01364 
01365 void Node::exitScheduler(CkQdMsg *msg) {
01366   //CmiPrintf("exitScheduler %d\n",CkMyPe());
01367   CsdExitScheduler();
01368   delete msg;
01369 }
01370 
01371 void Node::sendEnableEarlyExit(void) {
01372   CkQdMsg *msg = new CkQdMsg;
01373   CProxy_Node nodeProxy(thisgroup);
01374   nodeProxy[0].recvEnableEarlyExit(msg);
01375 }
01376 
01377 void Node::recvEnableEarlyExit(CkQdMsg *msg) {
01378   delete msg;
01379   enableEarlyExit();
01380 }
01381 
01382 void Node::enableEarlyExit(void) {
01383   if ( CkMyPe() ) {
01384     sendEnableEarlyExit();
01385   } else {
01386     CkStartQD(CkIndex_Node::earlyExit((CkQdMsg*)0),&thishandle);
01387   }
01388 }
01389 
01390 void Node::earlyExit(CkQdMsg *msg) {
01391   iout << iERROR << "Exiting prematurely; see error messages above.\n" << endi;
01392   if ( CmiNumPartitions() > 1 ) NAMD_quit("Exiting prematurely; see error messages above.");
01393   BackEnd::exit();
01394   delete msg;
01395 }
01396 
01397 
01398 //------------------------------------------------------------------------
01399 // Some odd utilities
01400 //------------------------------------------------------------------------
01401 void Node::saveMolDataPointers(NamdState *state)
01402 {
01403   this->molecule = state->molecule;
01404   this->parameters = state->parameters;
01405   this->simParameters = state->simParameters;
01406   this->configList = state->configList;
01407   this->pdb = state->pdb;
01408   this->state = state;
01409 }
01410 
01411 // entry methods for BG/P HPM (performance counters) library
01412 void Node::startHPM() {
01413 #if USE_HPM
01414   HPM_Start("500 steps", localRankOnNode);
01415 #endif
01416 }
01417 
01418 void Node::stopHPM() {
01419 #if USE_HPM
01420   HPM_Stop("500 steps", localRankOnNode);
01421   HPM_Print(CkMyPe(), localRankOnNode);
01422 #endif
01423 }
01424 
01425 void Node::traceBarrier(int turnOnTrace, int step){
01426         curTimeStep = step;
01427         if(turnOnTrace) traceBegin();
01428         else traceEnd();
01429 
01430     if(turnOnTrace) CmiTurnOnStats();
01431     else CmiTurnOffStats();
01432 
01433         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());    
01434         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01435         CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
01436         contribute(0, NULL, CkReduction::sum_int, cb);
01437         
01438 }
01439 
01440 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
01441         CmiAssert(CmiMyPe()==0);
01442         delete msg;     
01443         state->controller->resumeAfterTraceBarrier(curTimeStep);
01444 }
01445 
01446 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
01447 #ifdef MEASURE_NAMD_WITH_PAPI
01448         curMFlopStep = step;
01449         double results[NUM_PAPI_EVENTS+1];
01450 
01451         if(turnOnMeasure){              
01452           CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]=CmiWallTimer();
01453 
01454           long long counters[NUM_PAPI_EVENTS+1];
01455           int ret=PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
01456           if(ret==PAPI_OK)
01457             {
01458               //              CkPrintf("traceBarrier start counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01459             }
01460           else
01461             {
01462               CkPrintf("error PAPI_start_counters (%d) at step %d called on proc %d\n",ret , step, CkMyPe());
01463             }
01464           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)!=PAPI_OK)
01465             {
01466               CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01467             };
01468         }else{
01469           long long counters[NUM_PAPI_EVENTS+1];
01470           for(int i=0;i<NUM_PAPI_EVENTS;i++)  counters[i]=0LL;
01471           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)==PAPI_OK)
01472             {
01473 #if !MEASURE_PAPI_SPP
01474               results[0] = (double)counters[0]/1e6;
01475               results[1] = (double)counters[1]/1e6;
01476 #else
01477               for(int i=0;i<NUM_PAPI_EVENTS;i++)  results[i] = counters[i]/1e6;
01478 #endif
01479               //              for(int i=0;i<NUM_PAPI_EVENTS;i++) CkPrintf("[%d] counter %d is %ld\n",CkMyPe(),i,counters[i]);
01480             }
01481           else
01482             {
01483               //              CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01484             }
01485           //      CkPrintf("traceBarrier stop counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01486                 
01487           PAPI_stop_counters(counters, NUM_PAPI_EVENTS);        
01488         }
01489         if(CkMyPe()==0)
01490           //        CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01491         results[NUM_PAPI_EVENTS]=CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]; //starttime
01492         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01493         CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
01494         contribute(sizeof(double)*(NUM_PAPI_EVENTS+1), &results, CkReduction::sum_double, cb);  
01495 #endif
01496 }
01497 
01498 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
01499 #ifdef MEASURE_NAMD_WITH_PAPI
01500   
01501         if(simParameters->papiMeasureStartStep != curMFlopStep) {
01502                 double *results = (double *)msg->getData();
01503                 double endtime=CmiWallTimer();
01504                 int bstep = simParameters->papiMeasureStartStep;
01505                 int estep = bstep + simParameters->numPapiMeasureSteps;
01506 #if MEASURE_PAPI_SPP
01507                 CkPrintf("SPP INFO: PAPI_FP_OPS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[0]);
01508                 CkPrintf("SPP INFO: PAPI_TOT_INS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[1]);
01509                 CkPrintf("SPP INFO: perf::PERF_COUNT_HW_CACHE_LL:MISS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[2]);
01510                 CkPrintf("SPP INFO: DATA_PREFETCHER:ALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[3]);
01511                 CkPrintf("SPP INFO: PAPI_L1_DCA timestep %d to %d is %lf(1e6)\n", bstep,estep,results[4]);
01512                 CkPrintf("SPP INFO: PAPI_TOT_CYC timestep %d to % is %lf(1e6)\n", bstep,estep,results[5]);
01513                 //              CkPrintf("SPP INFO: INSTRUCTION_FETCH_STALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[6]);
01514                 //              CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]/CkNumPes());
01515                 CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]);
01516                 CkPrintf("SPP INFO: endtime %lf avgtime %lf tottime %lf\n", endtime,results[NUM_PAPI_EVENTS]/CkNumPes(),results[NUM_PAPI_EVENTS] );
01517 #else
01518                 if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
01519                         double totalFPIns = results[0];
01520                         if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
01521                         CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n", 
01522                                          bstep, estep, totalFPIns/CkNumPes());
01523                 }else{
01524                         char nameBuf[PAPI_MAX_STR_LEN];
01525                         CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ", 
01526                                          bstep, estep);
01527                         for(int i=0; i<NUM_PAPI_EVENTS; i++) {
01528                                 PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
01529                                 CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
01530                         }
01531                         CkPrintf("per processor\n");
01532                 }               
01533 #endif
01534         }
01535         delete msg;     
01536         state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
01537 #endif
01538 }
01539 
01540 extern char *gNAMDBinaryName;
01541 void Node::outputPatchComputeMaps(const char *filename, int tag){
01542         if(!simParameters->outputMaps && !simParameters->simulateInitialMapping) return;
01543 
01544         int numpes = CkNumPes();
01545         int nodesize = CkMyNodeSize();
01546         if(simParameters->simulateInitialMapping) {
01547                 numpes = simParameters->simulatedPEs;
01548                 nodesize = simParameters->simulatedNodeSize;
01549         }
01550 
01551         char fname[128];
01552         sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
01553 
01554         FILE *fp = fopen(fname, "w");
01555         if(fp == NULL) {
01556                 NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
01557                 return;
01558         }
01559         PatchMap *pMap = PatchMap::Object();
01560         ComputeMap *cMap = ComputeMap::Object();
01561         int numPatches = pMap->numPatches();
01562         int numComputes = cMap->numComputes();
01563         fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes, 
01564                         pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
01565         //output PatchMap info
01566         for(int i=0; i<numPatches; i++) {
01567         #ifdef MEM_OPT_VERSION
01568                 fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
01569         #else
01570                 fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
01571         #endif
01572         }
01573 
01574         //output ComputeMap info
01575         for(int i=0; i<numComputes; i++) {              
01576                 fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));             
01577         }
01578 }
01579 
01580 
01581 //======================================================================
01582 // Private functions
01583 
01584 #include "Node.def.h"
01585 

Generated on Sat Sep 23 01:17:14 2017 for NAMD by  doxygen 1.4.7