Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | File List | Namespace Members | Class Members | File Members

Node.C

Go to the documentation of this file.
00001 
00007 /*
00008    Toplevel routines for initializing a Node for a simulation
00009    one Node per Pe (processor element).
00010 */
00011 
00012 #if !defined(WIN32) || defined(__CYGWIN__)
00013 #include <unistd.h>
00014 #endif
00015 #include "InfoStream.h"
00016 #include "Node.decl.h"
00017 #include "Node.h"
00018 #ifdef DPMTA
00019 #include <pvm3.h>
00020 #endif
00021 
00022 #include "ProcessorPrivate.h"
00023 
00024 #define MIN_DEBUG_LEVEL 3
00025 //#define DEBUGM
00026 #include "Debug.h"
00027 
00028 #include <stdio.h>
00029 #include <converse.h>
00030 #include "memusage.h"
00031 #include "IMDOutput.h"
00032 #include "Lattice.h"
00033 #include "ComputeMsmMsa.h"  // needed for MsmMsaData definition
00034 #include "ComputeMsm.h"     // needed for MsmInitMsg definition
00035 #include "main.decl.h"
00036 #include "main.h"
00037 #include "WorkDistrib.h"
00038 #include "PatchMgr.h"
00039 #include "Patch.h"
00040 #include "Compute.h"
00041 #include "ComputeMap.h"
00042 #include "ComputeMgr.h"
00043 #include "Molecule.h"
00044 #include "HomePatchList.h"
00045 #include "AtomMap.h"
00046 #include "Sequencer.h"
00047 #include "Controller.h"
00048 #include "NamdState.h"
00049 #include "Output.h"
00050 #include "ProxyMgr.h"
00051 #include "PatchMap.h"
00052 #include "PatchMap.inl"
00053 #include "Parameters.h"
00054 #include "SimParameters.h"
00055 #include "Communicate.h"
00056 #include "LdbCoordinator.h"
00057 #include "ScriptTcl.h"
00058 #include "ComputeMgr.decl.h"
00059 #include "ComputePmeMgr.decl.h"
00060 #include "ComputeGridForceMgr.decl.h"
00061 #include "OptPmeMgr.decl.h"
00062 #include "Sync.h"
00063 #include "BackEnd.h"
00064 #include "PDB.h"
00065 #include "packmsg.h"
00066 #include "CollectionMgr.decl.h"
00067 #include "ParallelIOMgr.decl.h"
00068 // BEGIN LA
00069 #include "Random.h"
00070 // END LA
00071 
00072 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00073 extern "C" void CApplicationInit();
00074 #endif
00075 
00076 #include "DumpBench.h"
00077 
00078 
00079 #ifdef CMK_BALANCED_INJECTION_API
00080 #include "ckBIconfig.h"
00081 #endif
00082 
00083 #ifdef MEM_OPT_VERSION
00084 #include "CollectionMgr.h"
00085 #include "CollectionMaster.h"
00086 #include "CollectionMgr.decl.h"
00087 #include "CollectionMaster.decl.h"
00088 #endif
00089 
00090 #if USE_HPM
00091 extern "C" void HPM_Init(int);
00092 extern "C" void HPM_Start(char *label, int);
00093 extern "C" void HPM_Stop(char *label, int);
00094 extern "C" void HPM_Print(int, int);
00095 #endif
00096 
00097 #ifdef MEASURE_NAMD_WITH_PAPI
00098 #include "papi.h"
00099 #if CMK_SMP
00100 #include <pthread.h>
00101 #endif
00102 #define NUM_PAPI_EVENTS 6
00103 CkpvDeclare(int *, papiEvents);
00104 
00105 #define MEASURE_PAPI_SPP 1
00106 #define MEASURE_PAPI_CACHE 0
00107 #define MEASURE_PAPI_FLOPS 0
00108 
00109 static void namdInitPapiCounters(){
00110         if(CkMyRank()==0){
00111                 //only initialize per OS process (i.e. a charm node)
00112                 int retval = PAPI_library_init(PAPI_VER_CURRENT);
00113                 if(retval != PAPI_VER_CURRENT) {
00114                         if(CkMyPe()==0){
00115                                 CkPrintf("ERROR: PAPI library is not compatitible!");
00116                                 CkExit();
00117                         }
00118                 }
00119         #if CMK_SMP
00120                 //now only consider systems that are compatible with POSIX
00121                 if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
00122                         if(CkMyPe()==0){
00123                                 CkPrintf("ERROR: multi-thread mode in PAPI could not be initialized!");
00124                                 CkExit();
00125                         }
00126                 }
00127         #endif
00128         }
00129         CkpvInitialize(int *, papiEvents);
00130         CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS+1];
00131 
00132 #if MEASURE_PAPI_CACHE
00133         if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
00134                 CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
00135         }else{
00136                 if(CkMyPe()==0){
00137                         CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");                     
00138                 }
00139                 //if not default to PAPI_TOT_INS
00140                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00141         }
00142 
00143         if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
00144                 CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
00145         }else{
00146                 //if not default to PAPI_TOT_CYC
00147                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00148         }       
00149 #elif MEASURE_PAPI_FLOPS
00150         if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
00151                 CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
00152         }else{
00153                 if(CkMyPe()==0){
00154                         CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
00155                 }
00156                 //if not default to PAPI_TOT_INS
00157                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00158         }
00159 
00160         if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
00161                 CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
00162         }else{
00163                 //if not default to PAPI_TOT_CYC
00164                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00165         }
00166 #elif MEASURE_PAPI_SPP
00167 /* for SPP we record these
00168 1) PAPI_FP_OPS
00169 2) PAPI_TOT_INS
00170 3) perf::PERF_COUNT_HW_CACHE_LL:MISS
00171 4) DATA_PREFETCHER:ALL
00172 5) PAPI_L1_DCA
00173 6) INSTRUCTION_FETCH_STALL
00174 7) PAPI_TOT_CYC, and 
00175 8) real (wall) time
00176 */
00177         int papiEventSet = PAPI_NULL; 
00178         if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
00179           CmiAbort("PAPI failed to create event set!\n");
00180         }
00181 
00182         if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
00183                 CkpvAccess(papiEvents)[0] = PAPI_FP_OPS;
00184         }else{
00185                 if(CkMyPe()==0){
00186                         CkAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
00187                 }
00188         }
00189         if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
00190                 CkpvAccess(papiEvents)[1] = PAPI_TOT_INS;
00191         }else{
00192                 if(CkMyPe()==0){
00193                         CkAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
00194                 }
00195         }
00196         int EventCode;
00197         int ret;
00198         ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
00199         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00200           CkpvAccess(papiEvents)[2] = EventCode;
00201         }else{
00202                 if(CkMyPe()==0){
00203                         CkAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
00204                 }
00205         }
00206         ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
00207         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00208           CkpvAccess(papiEvents)[3] = EventCode;
00209         }else{
00210                 if(CkMyPe()==0){
00211                         CkAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
00212                 }
00213         }
00214         if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
00215                 CkpvAccess(papiEvents)[4] = PAPI_L1_DCA;
00216         }else{
00217                 if(CkMyPe()==0){
00218                         CkAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
00219                 }
00220         }
00221         /*      ret=PAPI_event_name_to_code("INSTRUCTION_FETCH_STALL",&EventCode);
00222         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00223           CkpvAccess(papiEvents)[5] = EventCode;
00224         }else{
00225                 if(CkMyPe()==0){
00226                         CkAbort("WARNING: INSTRUCTION_FETCH_STALL doesn't exist on this platform!");
00227                 }
00228         }
00229         */
00230         if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
00231                 CkpvAccess(papiEvents)[5] = PAPI_TOT_CYC;
00232         }else{
00233                 if(CkMyPe()==0){
00234                         CkAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
00235                 }
00236         }
00237         for(int i=0;i<NUM_PAPI_EVENTS;i++)
00238           {
00239             int papiRetValue=PAPI_add_events(papiEventSet, &CkpvAccess(papiEvents)[i],1);
00240             if (papiRetValue != PAPI_OK) {
00241               CkPrintf("failure for event %d\n",i);
00242               if (papiRetValue == PAPI_ECNFLCT) {
00243                 CmiAbort("PAPI events conflict! Please re-assign event types!\n");
00244               } else {
00245                 CmiAbort("PAPI failed to add designated events!\n");
00246               }
00247             }
00248             
00249           }
00250 #endif
00251 }
00252 #endif
00253 
00254 #ifdef OPENATOM_VERSION
00255 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
00256 {
00257   CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
00258 }
00259 #endif //OPENATOM_VERSION
00260 
00261 //======================================================================
00262 // Public Functions
00263 
00264 //----------------------------------------------------------------------
00265 
00266 int eventEndOfTimeStep;
00267 double startupTime;
00268 
00269 //----------------------------------------------------------------------
00270 // BOC constructor
00271 Node::Node(GroupInitMsg *msg)
00272 {    
00273   DebugM(4,"Creating Node\n");
00274 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00275   CApplicationInit();
00276 #endif
00277   if (CkpvAccess(Node_instance) == 0) {
00278     CkpvAccess(Node_instance) = this;
00279     eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep");
00280   } else {
00281     NAMD_bug("Node::Node() - another instance of Node exists!");
00282   }
00283 
00284   CkpvAccess(BOCclass_group) = msg->group;
00285   delete msg;
00286 
00287   CkpvAccess(BOCclass_group).node = thisgroup;
00288 
00289   startupPhase = 0;
00290 
00291   molecule = NULL;
00292   parameters = NULL;
00293   simParameters = NULL;
00294   configList = NULL;
00295   pdb = NULL;
00296   state = NULL;
00297   output = NULL;
00298   imd = new IMDOutput;
00299   colvars = 0;
00300 
00301 #if USE_HPM
00302   // assumes that this will be done only on BG/P
00303   TopoManager *tmgr = new TopoManager();
00304   int x, y, z;
00305   tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
00306   delete tmgr;
00307 #endif
00308 
00309   specialTracing = traceAvailable() && (traceIsOn()==0);
00310 
00311   DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
00312   patchMap = PatchMap::Instance();
00313   atomMap = AtomMap::Instance();
00314   if ( CkMyRank() == 0 ) ComputeMap::Instance();
00315 
00316   //Note: Binding BOC vars such as workDistrib has been moved
00317   //to the 1st phase of startup because the in-order message delivery
00318   //is not always guaranteed --Chao Mei
00319 #ifdef CMK_BALANCED_INJECTION_API
00320   if(CkMyRank() == 0){
00321     balancedInjectionLevel=ck_get_GNI_BIConfig();
00322     // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00323     ck_set_GNI_BIConfig(20);
00324     // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00325   }
00326 #endif
00327 
00328 }
00329 
00330 //----------------------------------------------------------------------
00331 // ~Node(void) needs to clean up everything.
00332 
00333 Node::~Node(void)
00334 {
00335   delete output;
00336   delete computeMap;
00337   delete atomMap;
00338   delete patchMap;
00339   delete CkpvAccess(comm);
00340   // BEGIN LA
00341   delete rand;
00342   // END LA
00343 #ifdef MEASURE_NAMD_WITH_PAPI
00344   delete CkpvAccess(papiEvents);
00345 #endif
00346 }
00347 
00348 void Node::bindBocVars(){
00349     DebugM(4,"Binding to BOC's\n");
00350     CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00351     patchMgr = pm.ckLocalBranch();
00352     CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
00353     proxyMgr = prm.ckLocalBranch();
00354     CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
00355     workDistrib = wd.ckLocalBranch();
00356     CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00357     computeMgr = cm.ckLocalBranch();
00358     CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
00359     ldbCoordinator = lc.ckLocalBranch();
00360   #ifdef MEM_OPT_VERSION      
00361     CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
00362     ioMgr = io.ckLocalBranch();
00363   #endif
00364 
00365 }
00366 
00367 //----------------------------------------------------------------------
00368 // Malloc Test Sequence
00369 void Node::mallocTest(int step) {
00370   int MB = 1024*1024;
00371   int size = 100;
00372   char* foo = (char*) malloc(size*MB);
00373   if ( ! foo ) {
00374     char buf[256];
00375     sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
00376     NAMD_die(buf);
00377   }
00378   memset(foo,0,size*MB*sizeof(char));
00379 }
00380 
00381 void Node::mallocTestQd(CkQdMsg *qmsg) {
00382   delete qmsg;
00383   if ( mallocTest_size ) {
00384     CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
00385   } else {
00386     CkPrintf("Starting malloc test on all PEs.\n");
00387   }
00388   fflush(stdout);
00389   ++mallocTest_size;
00390   CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00391   (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
00392 }
00393 
00394 //----------------------------------------------------------------------
00395 // Startup Sequence
00396 
00397 void Node::messageStartUp() {
00398   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00399 }
00400 
00401 void Node::startUp(CkQdMsg *qmsg) {
00402   delete qmsg;
00403   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00404 }
00405 
00406 SimParameters *node_simParameters;
00407 Parameters *node_parameters;
00408 Molecule *node_molecule;
00409 
00410 extern void registerUserEventsForAllComputeObjs(void);
00411 
00412 void Node::startup() {
00413   int gotoRun = false;
00414   double newTime;
00415 
00416   if (!CkMyPe()) {
00417     if (!startupPhase) {
00418       iout << iINFO << "\n";
00419       startupTime = CmiWallTimer();
00420       iout << iINFO << "Entering startup at " << startupTime << " s, ";
00421     } else {
00422       newTime = CmiWallTimer();
00423       iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00424            << newTime - startupTime << " s, ";
00425       startupTime = newTime;
00426     }
00427     iout << memusage_MB() << " MB of memory in use\n" << endi;
00428     fflush(stdout);
00429   }
00430   switch (startupPhase) {
00431 
00432   case 0:
00433     computeMap = ComputeMap::Object();
00434     namdOneCommInit(); // Namd1.X style
00435   break;
00436 
00437   case 1:
00438       bindBocVars();
00439 
00440     // send & receive molecule, simparameters... (Namd1.X style)
00441     if (CkMyPe()) {
00442       namdOneRecv();
00443     } else {
00444       namdOneSend();
00445     }
00446   break;
00447 
00448   case 2:
00449     // fix up one-per-node objects (for SMP version)
00450     simParameters = node_simParameters;
00451     parameters = node_parameters;
00452     molecule = node_molecule;
00453     
00454     #if !CMK_SMP || ! USE_CKLOOP
00455     //the CkLoop library should be only used in SMP mode
00456     simParameters->useCkLoop = 0;
00457     #endif
00458 
00459 
00460     if ( simParameters->mallocTest ) {
00461       if (!CkMyPe()) {
00462         mallocTest_size = 0;
00463         CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00464       }
00465       return;
00466     }
00467 
00468       
00469         #ifdef MEASURE_NAMD_WITH_PAPI
00470         if(simParameters->papiMeasure) namdInitPapiCounters();  
00471         #endif
00472     
00473     #ifdef MEM_OPT_VERSION
00474     //At this point, each Node object has received the simParameters,
00475     //parameters and the atom signatures info from the master Node
00476     //(proc 0). It's time to initialize the parallel IO manager and
00477     //read the binary per-atom file --Chao Mei
00478 
00479     //Step 1: initialize the parallel IO manager per Node
00480     ioMgr->initialize(this);
00481     #endif
00482 
00483   break;
00484 
00485   case 3:
00486 
00487     #ifdef MEM_OPT_VERSION
00488     //Step 2: read the binary per-atom files (signater index, coordinates etc.)
00489     ioMgr->readPerAtomInfo();
00490     #endif
00491 
00492   break;
00493 
00494   case 4:
00495 
00496     #ifdef MEM_OPT_VERSION
00497     //Step 3: update counters of tuples and exclusions inside Molecule object
00498     ioMgr->updateMolInfo();
00499 
00500     //Step 4: prepare distributing the atoms to neighboring procs if necessary
00501     ioMgr->migrateAtomsMGrp();
00502 
00503     //step 5: initialize patchMap and send it to every other processors
00504     //to decide atoms to patch distribution on every input processor
00505     if(!CkMyPe()) {
00506         workDistrib->patchMapInit(); // create space division
00507         workDistrib->sendPatchMap();
00508     }
00509     #endif
00510 
00511     #if USE_HPM
00512     HPM_Init(localRankOnNode);
00513     #endif    
00514 
00515     // take care of inital thread setting
00516     threadInit();
00517 
00518     // create blank AtomMap
00519     AtomMap::Object()->allocateMap(molecule->numAtoms);
00520 
00521     if (!CkMyPe()) {
00522       if (simParameters->useOptPME)
00523         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_OptPmeMgr::ckNew();
00524       else 
00525         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
00526         #ifdef OPENATOM_VERSION
00527         if ( simParameters->openatomOn ) { 
00528           CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
00529         }
00530         #endif // OPENATOM_VERSION
00531 
00532     }
00533     
00534     #ifdef OPENATOM_VERSION
00535     if ( simParameters->openatomOn ) {
00536       // if ( ! CkMyPe() ) { 
00537         CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle); 
00538         startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
00539       // }
00540     }
00541     #endif // OPENATOM_VERSION
00542   
00543     // BEGIN LA
00544     rand = new Random(simParameters->randomSeed);
00545     rand->split(CkMyPe(), CkNumPes());
00546     // END LA
00547 
00548   break;
00549 
00550   case 5:
00551     #ifdef MEM_OPT_VERSION
00552     //Now, every input proc has received all the atoms necessary
00553     //to decide the patches those atoms belong to
00554     
00555     //step 1: integrate the migrated atoms into the atom list that
00556     //contains the initally distributed atoms, and sort the atoms
00557     //based on hydrogenList value
00558     ioMgr->integrateMigratedAtoms();
00559 
00560     //step 2: integrate the cluster size of each atom on each output proc
00561     ioMgr->integrateClusterSize();
00562 
00563     //step 3: calculate the number of atoms in each patch on every
00564     //input procs (atoms belonging to a patch may lie on different
00565     //procs), and reduce such info on proc 0. Such info is required
00566     //for determing which node a particular patch is assigned to.
00567     ioMgr->calcAtomsInEachPatch();
00568 
00569     //set to false to re-send PatchMap later
00570     workDistrib->setPatchMapArrived(false);
00571     #endif
00572     break;
00573   case 6:     
00574     if(simParameters->isSendSpanningTreeOn()) {                         
00575                         ProxyMgr::Object()->setSendSpanning();
00576                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);                             
00577     }
00578     if(simParameters->isRecvSpanningTreeOn()) {                         
00579                         ProxyMgr::Object()->setRecvSpanning();
00580                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);                             
00581     }
00582     #ifdef PROCTRACE_DEBUG
00583     DebugFileTrace::Instance("procTrace");
00584     #endif
00585 
00586     if (!CkMyPe()) {
00587       output = new Output; // create output object just on PE(0)
00588 
00589       #ifndef MEM_OPT_VERSION
00590       workDistrib->patchMapInit(); // create space division
00591       workDistrib->createHomePatches(); // load atoms into HomePatch(es)
00592       #endif
00593       
00594       workDistrib->assignNodeToPatch();
00595       workDistrib->mapComputes();         
00596       //ComputeMap::Object()->printComputeMap();
00597           
00598           if(simParameters->simulateInitialMapping) {
00599           iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
00600                           << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
00601                   outputPatchComputeMaps("init_mapping", 0);
00602                   iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
00603                   CkExit();
00604           }
00605 
00606       registerUserEventsForAllComputeObjs();
00607 
00608       //in MEM_OPT_VERSION, patchMap is resent
00609       //because they have been updated since creation including
00610       //#atoms per patch, the proc a patch should stay etc. --Chao Mei
00611       workDistrib->sendPatchMap();
00612       #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00613       CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00614       //a node broadcast
00615       npm.createProxyInfo(PatchMap::Object()->numPatches());
00616       #endif
00617     }
00618     {
00619         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00620         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00621         if(CkMyRank()==0) {
00622             //just need to register once
00623             npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
00624         }
00625         npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
00626         #endif
00627     }
00628   break;
00629 
00630   case 7:
00631     if ( simParameters->PMEOn ) {
00632       if ( simParameters->useOptPME ) {
00633         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00634         pme[CkMyPe()].initialize(new CkQdMsg);
00635       }
00636       else {
00637         #ifdef OPENATOM_VERSION
00638         if ( simParameters->openatomOn ) { 
00639           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00640           moa[CkMyPe()].initialize(new CkQdMsg);
00641         }
00642         #endif // OPENATOM_VERSION
00643         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00644         pme[CkMyPe()].initialize(new CkQdMsg);
00645       }
00646     }
00647 #ifdef CHARM_HAS_MSA
00648     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00649       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00650       msm[CkMyPe()].initialize(new CkQdMsg);
00651     }
00652 #else
00653     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00654       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00655       MsmInitMsg *msg = new MsmInitMsg;
00656       Lattice lattice = simParameters->lattice;  // system lattice vectors
00657       ScaledPosition smin=0, smax=0;
00658       if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
00659         msg->smin = smin;
00660         msg->smax = smax;
00661         msm[CkMyPe()].initialize(msg);  // call from my own PE
00662       }
00663       else if ( ! CkMyPe() ) {
00664         pdb->get_extremes(smin, smax);  // only available on PE 0
00665         msg->smin = smin;
00666         msg->smax = smax;
00667         msm.initialize(msg);  // broadcast to chare group
00668       }
00669 
00670       /*
00671       CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00672       Node *node = nd.ckLocalBranch();
00673       ScaledPosition smin, smax;
00674       node->pdb->get_extremes(smin, smax);
00675       msg->smin = smin;                       // extreme positions in system
00676       msg->smax = smax;
00677       msm[CkMyPe()].initialize(msg);
00678       */
00679     }
00680 #endif
00681 
00682     if (!CkMyPe()) {
00683       workDistrib->sendComputeMap();
00684     }
00685 
00686     #ifdef MEM_OPT_VERSION
00687     //migrate atoms to HomePatch processors
00688     ioMgr->sendAtomsToHomePatchProcs();
00689     #endif
00690     break;
00691     
00692   case 8:
00693     if ( simParameters->PMEOn ) {
00694       if ( simParameters->useOptPME ) {
00695         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00696         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00697       }
00698       else {
00699         #ifdef OPENATOM_VERSION
00700         if ( simParameters->openatomOn ) { 
00701           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00702           moa[CkMyPe()].initWorkers(new CkQdMsg);
00703         }
00704         #endif // OPENATOM_VERSION
00705         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00706         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00707       }
00708     }
00709 #ifdef CHARM_HAS_MSA
00710     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00711       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00712       msm[CkMyPe()].initWorkers(new CkQdMsg);
00713     }
00714 #else
00715     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00716       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00717       msm[CkMyPe()].update(new CkQdMsg);
00718     }
00719 #endif
00720 
00721     #ifdef MEM_OPT_VERSION
00722     //Now every processor has all the atoms it needs to create the HomePatches.
00723     //The HomePatches are created in parallel on every home patch procs.
00724     ioMgr->createHomePatches();
00725     #else
00726     if (!CkMyPe()) {
00727       workDistrib->distributeHomePatches();          
00728     }
00729     #endif
00730   break;
00731 
00732   case 9:
00733     if ( simParameters->PMEOn ) {
00734       if ( simParameters->useOptPME ) {
00735         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00736         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00737       }
00738       else {
00739         #ifdef OPENATOM_VERSION
00740         if ( simParameters->openatomOn ) { 
00741           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00742           moa[CkMyPe()].startWorkers(new CkQdMsg);
00743         }
00744         #endif // OPENATOM_VERSION
00745         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00746         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00747       }
00748     }
00749 #ifdef CHARM_HAS_MSA
00750     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00751       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00752       msm[CkMyPe()].startWorkers(new CkQdMsg);
00753     }
00754 #else
00755     /*
00756     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00757       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00758       //msm[CkMyPe()].startWorkers(new CkQdMsg);
00759     }
00760     */
00761 #endif
00762 
00763     proxyMgr->createProxies();  // need Home patches before this
00764     if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
00765 
00766 #ifdef NAMD_TCL
00767     // TclInitSubsystems() has a race condition so we create one interp per node here
00768     if (CkMyPe() && ! CkMyRank()) Tcl_DeleteInterp(Tcl_CreateInterp());
00769 #endif
00770 
00771 #ifdef USE_NODEPATCHMGR
00772         //at this point, PatchMap info has been recved on PEs. It is time to create
00773         //the home patch spanning tree for receiving proxy list info
00774         if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
00775                 if(CkMyRank()==0) {
00776                         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00777                         npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
00778                 }
00779         }
00780 #endif
00781 
00782   break;
00783 
00784   case 10:
00785     if (!CkMyPe()) {
00786       iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
00787            << " COMPUTE OBJECTS\n" << endi;
00788     }
00789     DebugM(4,"Creating Computes\n");
00790     computeMgr->createComputes(ComputeMap::Object());
00791     DebugM(4,"Building Sequencers\n");
00792     buildSequencers();
00793     DebugM(4,"Initializing LDB\n");
00794     LdbCoordinator::Object()->initialize(PatchMap::Object(),ComputeMap::Object());
00795   break;
00796 
00797   case 11:
00798     // computes may create proxies on the fly so put these in separate phase
00799     Sync::Object()->openSync();  // decide if to open local Sync 
00800     if (proxySendSpanning || proxyRecvSpanning ) proxyMgr->buildProxySpanningTree();
00801 #ifdef CMK_BALANCED_INJECTION_API
00802     if(CkMyRank() == 0){
00803       // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00804       ck_set_GNI_BIConfig(balancedInjectionLevel);
00805       // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00806     }
00807 #endif
00808 
00809   break;
00810 
00811   case 12:
00812     {
00813         //For debugging
00814         /*if(!CkMyPe()){
00815         FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
00816         dumpbench(dumpFile);
00817         NAMD_die("Normal execution\n");
00818         }*/
00819     }
00820     #ifdef MEM_OPT_VERSION
00821     //free space in the Molecule object that are not used anymore
00822     ioMgr->freeMolSpace();
00823     #endif
00824     gotoRun = true;
00825   break;
00826 
00827   default:
00828     NAMD_bug("Startup Phase has a bug - check case statement");
00829   break;
00830 
00831   }
00832 
00833   startupPhase++;
00834   if (!CkMyPe()) {
00835     if (!gotoRun) {
00836       CkStartQD(CkIndex_Node::startUp((CkQdMsg*)0),&thishandle);
00837     } else {
00838       Node::messageRun();
00839     }
00840   }
00841 }
00842 
00843 #ifdef OPENATOM_VERSION
00844 void Node::doneMoaStart()
00845 {
00846 #ifdef OPENATOM_VERSION_DEBUG
00847   CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
00848 #endif //OPENATOM_VERSION_DEBUG
00849 }
00850 #endif //OPENATOM_VERSION
00851 
00852 void Node::namdOneCommInit()
00853 {
00854   if (CkpvAccess(comm) == NULL) {
00855     CkpvAccess(comm) = new Communicate();
00856 #ifdef DPMTA
00857     pvmc_init();
00858 #endif
00859   }
00860 }
00861 
00862 // Namd 1.X style Send/Recv of simulation information
00863 
00864 void Node::namdOneRecv() {
00865   if ( CmiMyRank() ) return;
00866 
00867   MIStream *conv_msg;
00868 
00869   // Receive molecule and simulation parameter information
00870   simParameters = node_simParameters = new SimParameters;
00871   //****** BEGIN CHARMM/XPLOR type changes
00872   parameters = node_parameters = new Parameters();
00873   //****** END CHARMM/XPLOR type changes
00874   molecule = node_molecule = new Molecule(simParameters,parameters);
00875 
00876   DebugM(4, "Getting SimParameters\n");
00877   conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
00878   simParameters->receive_SimParameters(conv_msg);
00879 
00880   DebugM(4, "Getting Parameters\n");
00881   conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
00882   parameters->receive_Parameters(conv_msg);
00883 
00884   DebugM(4, "Getting Molecule\n");
00885   conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00886   // Modified by JLai -- 10.21.11
00887   molecule->receive_Molecule(conv_msg);
00888   if(simParameters->goForcesOn) {
00889     iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
00890     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00891     molecule->receive_GoMolecule(conv_msg);
00892   } 
00893   // End of modification
00894   DebugM(4, "Done Receiving\n");
00895 }
00896 
00897 void Node::namdOneSend() {
00898   node_simParameters = simParameters;
00899   node_parameters = parameters;
00900   node_molecule = molecule;
00901 
00902   MOStream *conv_msg;
00903   // I'm Pe(0) so I send what I know
00904   DebugM(4, "Sending SimParameters\n");  
00905   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
00906   simParameters->send_SimParameters(conv_msg);
00907 
00908   DebugM(4, "Sending Parameters\n");
00909   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
00910   parameters->send_Parameters(conv_msg);
00911 
00912   DebugM(4, "Sending Molecule\n");
00913   int bufSize = BUFSIZE;
00914   if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
00915   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
00916   // Modified by JLai -- 10.21.11
00917   molecule->send_Molecule(conv_msg);
00918   
00919   if(simParameters->goForcesOn) {
00920     iout << iINFO <<  "Master Node sending GoMolecule Information" << "\n" << endi;
00921     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
00922     molecule->send_GoMolecule(conv_msg);
00923   } // End of modification
00924 }
00925 
00926 // Initial thread setup
00927 
00928 void Node::threadInit() {
00929   // Thread initialization
00930   if (CthImplemented()) {
00931     CthSetStrategyDefault(CthSelf());
00932   } else {
00933     NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
00934   }
00935 }
00936 
00937 //
00938 void Node::buildSequencers() {
00939   HomePatchList *hpl = PatchMap::Object()->homePatchList();
00940   ResizeArrayIter<HomePatchElem> ai(*hpl);
00941 
00942   // Controller object is only on Pe(0)
00943   if ( ! CkMyPe() ) {
00944     Controller *controller = new Controller(state);
00945     state->useController(controller);
00946   }
00947 
00948   // Assign Sequencer to all HomePatch(es)
00949   for (ai=ai.begin(); ai != ai.end(); ai++) {
00950     HomePatch *patch = (*ai).patch;
00951     Sequencer *sequencer = new Sequencer(patch);
00952     patch->useSequencer(sequencer);
00953   }
00954 }
00955 
00956 
00957 
00958 //-----------------------------------------------------------------------
00959 // Node run() - broadcast to all nodes
00960 //-----------------------------------------------------------------------
00961 void Node::messageRun() {
00962   (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
00963 }
00964 
00965 
00966 //-----------------------------------------------------------------------
00967 // run(void) runs the specified simulation for the specified number of
00968 // steps, overriding the contents of the configuration file
00969 //-----------------------------------------------------------------------
00970 void Node::run()
00971 {
00972   // Start Controller (aka scalar Sequencer) on Pe(0)
00973 //  printf("\n\n I am in Node.C in run method about to call  state->runController\n\n");
00974   if ( ! CkMyPe() ) {
00975     state->runController();
00976   }
00977 
00978   DebugM(4, "Starting Sequencers\n");
00979   // Run Sequencer on each HomePatch - i.e. start simulation
00980   HomePatchList *hpl = PatchMap::Object()->homePatchList();
00981   ResizeArrayIter<HomePatchElem> ai(*hpl);
00982   for (ai=ai.begin(); ai != ai.end(); ai++) {
00983     HomePatch *patch = (*ai).patch;
00984 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
00985     patch->runSequencer();
00986   }
00987 
00988   if (!CkMyPe()) {
00989     double newTime = CmiWallTimer();
00990     iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00991          << newTime - startupTime << " s, "
00992          << memusage_MB() << " MB of memory in use\n";
00993     iout << iINFO << "Finished startup at " << newTime << " s, "
00994          << memusage_MB() << " MB of memory in use\n\n" << endi;
00995     fflush(stdout);
00996   }
00997   
00998 }
00999 
01000 
01001 //-----------------------------------------------------------------------
01002 // Node scriptBarrier() - twiddle parameters with simulation halted
01003 //-----------------------------------------------------------------------
01004 
01005 void Node::enableScriptBarrier() {
01006   CkStartQD(CkIndex_Node::scriptBarrier((CkQdMsg*)0),&thishandle);
01007 }
01008 
01009 void Node::scriptBarrier(CkQdMsg *qmsg) {
01010   delete qmsg;
01011   //script->awaken();
01012 }
01013 
01014 void Node::scriptParam(ScriptParamMsg *msg) {
01015   simParameters->scriptSet(msg->param,msg->value);
01016   delete msg;
01017 }
01018 
01019 void Node::reloadCharges(const char *filename) {
01020   FILE *file = fopen(filename,"r");
01021   if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
01022 
01023   int n = molecule->numAtoms;
01024   float *charge = new float[n];
01025 
01026   for ( int i = 0; i < n; ++i ) {
01027     if ( ! fscanf(file,"%f",&charge[i]) )
01028       NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
01029   }
01030 
01031   fclose(file);
01032   CProxy_Node(thisgroup).reloadCharges(charge,n);
01033   delete [] charge;
01034 }
01035 
01036 void Node::reloadCharges(float charge[], int n) {
01037   molecule->reloadCharges(charge,n);
01038 }
01039 
01040 
01041 // BEGIN gf
01042 void Node::reloadGridforceGrid(const char * key) {
01043     DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
01044     
01045     int gridnum;
01046     MGridforceParams *mgridParams;
01047     if (key == NULL) {
01048         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01049         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01050     } else {
01051         gridnum = simParameters->mgridforcelist.index_for_key(key);
01052         mgridParams = simParameters->mgridforcelist.find_key(key);
01053     }
01054     
01055     if (gridnum < 0 || mgridParams == NULL) {
01056         NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
01057     }
01058     
01059     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01060     if (grid == NULL) {
01061         NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
01062     }
01063     grid->reinitialize(simParameters, mgridParams);
01064     
01065     CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
01066     
01067     DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
01068 }
01069 
01070 void Node::reloadGridforceGrid(int gridnum) {
01071     DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
01072     
01073     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01074     if (grid == NULL) {
01075         NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
01076     }
01077     
01078     if (CkMyPe()) {
01079         // not node 0 -> receive grid
01080         if (CmiMyRank()) return;
01081         
01082         DebugM(4, "Receiving grid\n");
01083         
01084         delete grid;
01085         
01086         MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
01087         grid = GridforceGrid::unpack_grid(gridnum, msg);
01088         molecule->set_gridfrc_grid(gridnum, grid);
01089         delete msg;
01090     } else {
01091         // node 0 -> send grid
01092         DebugM(4, "Sending grid\n");
01093         
01094         MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
01095         GridforceGrid::pack_grid(grid, msg);
01096         msg->end();
01097         delete msg;
01098     }
01099     
01100     DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
01101 }
01102 // END gf
01103 
01104 
01105 void Node::sendEnableExitScheduler(void) {
01106   //CmiPrintf("sendEnableExitScheduler\n");
01107   CkQdMsg *msg = new CkQdMsg;
01108   CProxy_Node nodeProxy(thisgroup);
01109   nodeProxy[0].recvEnableExitScheduler(msg);
01110 }
01111 
01112 void Node::recvEnableExitScheduler(CkQdMsg *msg) {
01113   //CmiPrintf("recvEnableExitScheduler\n");
01114   delete msg;
01115   enableExitScheduler();
01116 }
01117 
01118 void Node::enableExitScheduler(void) {
01119   if ( CkMyPe() ) {
01120     sendEnableExitScheduler();
01121   } else {
01122     CkStartQD(CkIndex_Node::exitScheduler((CkQdMsg*)0),&thishandle);
01123   }
01124 }
01125 
01126 void Node::exitScheduler(CkQdMsg *msg) {
01127   //CmiPrintf("exitScheduler %d\n",CkMyPe());
01128   CsdExitScheduler();
01129   delete msg;
01130 }
01131 
01132 void Node::sendEnableEarlyExit(void) {
01133   CkQdMsg *msg = new CkQdMsg;
01134   CProxy_Node nodeProxy(thisgroup);
01135   nodeProxy[0].recvEnableEarlyExit(msg);
01136 }
01137 
01138 void Node::recvEnableEarlyExit(CkQdMsg *msg) {
01139   delete msg;
01140   enableEarlyExit();
01141 }
01142 
01143 void Node::enableEarlyExit(void) {
01144   if ( CkMyPe() ) {
01145     sendEnableEarlyExit();
01146   } else {
01147     CkStartQD(CkIndex_Node::earlyExit((CkQdMsg*)0),&thishandle);
01148   }
01149 }
01150 
01151 void Node::earlyExit(CkQdMsg *msg) {
01152   iout << iERROR << "Exiting prematurely; see error messages above.\n" << endi;
01153   BackEnd::exit();
01154   delete msg;
01155 }
01156 
01157 
01158 //------------------------------------------------------------------------
01159 // Some odd utilities
01160 //------------------------------------------------------------------------
01161 void Node::saveMolDataPointers(NamdState *state)
01162 {
01163   this->molecule = state->molecule;
01164   this->parameters = state->parameters;
01165   this->simParameters = state->simParameters;
01166   this->configList = state->configList;
01167   this->pdb = state->pdb;
01168   this->state = state;
01169 }
01170 
01171 // entry methods for BG/P HPM (performance counters) library
01172 void Node::startHPM() {
01173 #if USE_HPM
01174   HPM_Start("500 steps", localRankOnNode);
01175 #endif
01176 }
01177 
01178 void Node::stopHPM() {
01179 #if USE_HPM
01180   HPM_Stop("500 steps", localRankOnNode);
01181   HPM_Print(CkMyPe(), localRankOnNode);
01182 #endif
01183 }
01184 
01185 void Node::traceBarrier(int turnOnTrace, int step){
01186         curTimeStep = step;
01187         if(turnOnTrace) traceBegin();
01188         else traceEnd();
01189 
01190 #if CHARM_VERSION > 60400
01191     if(turnOnTrace) CmiTurnOnStats();
01192     else CmiTurnOffStats();
01193 #endif
01194 
01195         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());    
01196         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01197         CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
01198         contribute(0, NULL, CkReduction::sum_int, cb);
01199         
01200 }
01201 
01202 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
01203         CmiAssert(CmiMyPe()==0);
01204         delete msg;     
01205         state->controller->resumeAfterTraceBarrier(curTimeStep);
01206 }
01207 
01208 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
01209 #ifdef MEASURE_NAMD_WITH_PAPI
01210         curMFlopStep = step;
01211         double results[NUM_PAPI_EVENTS+1];
01212 
01213         if(turnOnMeasure){              
01214           CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]=CmiWallTimer();
01215 
01216           long long counters[NUM_PAPI_EVENTS+1];
01217           int ret=PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
01218           if(ret==PAPI_OK)
01219             {
01220               //              CkPrintf("traceBarrier start counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01221             }
01222           else
01223             {
01224               CkPrintf("error PAPI_start_counters (%d) at step %d called on proc %d\n",ret , step, CkMyPe());
01225             }
01226           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)!=PAPI_OK)
01227             {
01228               CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01229             };
01230         }else{
01231           long long counters[NUM_PAPI_EVENTS+1];
01232           for(int i=0;i<NUM_PAPI_EVENTS;i++)  counters[i]=0LL;
01233           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)==PAPI_OK)
01234             {
01235 #if !MEASURE_PAPI_SPP
01236               results[0] = (double)counters[0]/1e6;
01237               results[1] = (double)counters[1]/1e6;
01238 #else
01239               for(int i=0;i<NUM_PAPI_EVENTS;i++)  results[i] = counters[i]/1e6;
01240 #endif
01241               //              for(int i=0;i<NUM_PAPI_EVENTS;i++) CkPrintf("[%d] counter %d is %ld\n",CkMyPe(),i,counters[i]);
01242             }
01243           else
01244             {
01245               //              CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01246             }
01247           //      CkPrintf("traceBarrier stop counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01248                 
01249           PAPI_stop_counters(counters, NUM_PAPI_EVENTS);        
01250         }
01251         if(CkMyPe()==0)
01252           //        CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01253         results[NUM_PAPI_EVENTS]=CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]; //starttime
01254         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01255         CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
01256         contribute(sizeof(double)*(NUM_PAPI_EVENTS+1), &results, CkReduction::sum_double, cb);  
01257 #endif
01258 }
01259 
01260 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
01261 #ifdef MEASURE_NAMD_WITH_PAPI
01262   
01263         if(simParameters->papiMeasureStartStep != curMFlopStep) {
01264                 double *results = (double *)msg->getData();
01265                 double endtime=CmiWallTimer();
01266                 int bstep = simParameters->papiMeasureStartStep;
01267                 int estep = bstep + simParameters->numPapiMeasureSteps;
01268 #if MEASURE_PAPI_SPP
01269                 CkPrintf("SPP INFO: PAPI_FP_OPS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[0]);
01270                 CkPrintf("SPP INFO: PAPI_TOT_INS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[1]);
01271                 CkPrintf("SPP INFO: perf::PERF_COUNT_HW_CACHE_LL:MISS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[2]);
01272                 CkPrintf("SPP INFO: DATA_PREFETCHER:ALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[3]);
01273                 CkPrintf("SPP INFO: PAPI_L1_DCA timestep %d to %d is %lf(1e6)\n", bstep,estep,results[4]);
01274                 CkPrintf("SPP INFO: PAPI_TOT_CYC timestep %d to % is %lf(1e6)\n", bstep,estep,results[5]);
01275                 //              CkPrintf("SPP INFO: INSTRUCTION_FETCH_STALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[6]);
01276                 //              CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]/CkNumPes());
01277                 CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]);
01278                 CkPrintf("SPP INFO: endtime %lf avgtime %lf tottime %lf\n", endtime,results[NUM_PAPI_EVENTS]/CkNumPes(),results[NUM_PAPI_EVENTS] );
01279 #else
01280                 if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
01281                         double totalFPIns = results[0];
01282                         if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
01283                         CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n", 
01284                                          bstep, estep, totalFPIns/CkNumPes());
01285                 }else{
01286                         char nameBuf[PAPI_MAX_STR_LEN];
01287                         CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ", 
01288                                          bstep, estep);
01289                         for(int i=0; i<NUM_PAPI_EVENTS; i++) {
01290                                 PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
01291                                 CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
01292                         }
01293                         CkPrintf("per processor\n");
01294                 }               
01295 #endif
01296         }
01297         delete msg;     
01298         state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
01299 #endif
01300 }
01301 
01302 extern char *gNAMDBinaryName;
01303 void Node::outputPatchComputeMaps(const char *filename, int tag){
01304         if(!simParameters->outputMaps && !simParameters->simulateInitialMapping) return;
01305 
01306         int numpes = CkNumPes();
01307         int nodesize = CkMyNodeSize();
01308         if(simParameters->simulateInitialMapping) {
01309                 numpes = simParameters->simulatedPEs;
01310                 nodesize = simParameters->simulatedNodeSize;
01311         }
01312 
01313         char fname[128];
01314         sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
01315 
01316         FILE *fp = fopen(fname, "w");
01317         if(fp == NULL) {
01318                 NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
01319                 return;
01320         }
01321         PatchMap *pMap = PatchMap::Object();
01322         ComputeMap *cMap = ComputeMap::Object();
01323         int numPatches = pMap->numPatches();
01324         int numComputes = cMap->numComputes();
01325         fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes, 
01326                         pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
01327         //output PatchMap info
01328         for(int i=0; i<numPatches; i++) {
01329         #ifdef MEM_OPT_VERSION
01330                 fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
01331         #else
01332                 fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
01333         #endif
01334         }
01335 
01336         //output ComputeMap info
01337         for(int i=0; i<numComputes; i++) {              
01338                 fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));             
01339         }
01340 }
01341 
01342 
01343 //======================================================================
01344 // Private functions
01345 
01346 #include "Node.def.h"
01347 

Generated on Wed Jun 19 04:08:18 2013 for NAMD by  doxygen 1.3.9.1