Node.C

Go to the documentation of this file.
00001 
00007 /*
00008    Toplevel routines for initializing a Node for a simulation
00009    one Node per Pe (processor element).
00010 */
00011 
00012 #if !defined(WIN32) || defined(__CYGWIN__)
00013 #include <unistd.h>
00014 #endif
00015 #include "InfoStream.h"
00016 #include "Node.decl.h"
00017 #include "Node.h"
00018 #ifdef DPMTA
00019 #include <pvm3.h>
00020 #endif
00021 
00022 #include "ProcessorPrivate.h"
00023 
00024 #define MIN_DEBUG_LEVEL 3
00025 //#define DEBUGM
00026 #include "Debug.h"
00027 
00028 #include <stdio.h>
00029 #include <converse.h>
00030 #include "memusage.h"
00031 #include "IMDOutput.h"
00032 #include "Lattice.h"
00033 #include "ComputeMsmMsa.h"  // needed for MsmMsaData definition
00034 #include "ComputeMsm.h"     // needed for MsmInitMsg definition
00035 #include "main.decl.h"
00036 #include "main.h"
00037 #include "WorkDistrib.h"
00038 #include "PatchMgr.h"
00039 #include "Patch.h"
00040 #include "Compute.h"
00041 #include "ComputeMap.h"
00042 #include "ComputeMgr.h"
00043 #include "Molecule.h"
00044 #include "HomePatchList.h"
00045 #include "AtomMap.h"
00046 #include "Sequencer.h"
00047 #include "Controller.h"
00048 #include "NamdState.h"
00049 #include "Output.h"
00050 #include "ProxyMgr.h"
00051 #include "PatchMap.h"
00052 #include "PatchMap.inl"
00053 #include "Parameters.h"
00054 #include "SimParameters.h"
00055 #include "Communicate.h"
00056 #include "LdbCoordinator.h"
00057 #include "ScriptTcl.h"
00058 #include "ComputeMgr.decl.h"
00059 #include "ComputePmeMgr.decl.h"
00060 // #ifdef NAMD_CUDA
00061 #include "ComputeCUDAMgr.decl.h"
00062 #include "ComputeCUDAMgr.h"
00063 #include "ComputePmeCUDAMgr.decl.h"
00064 #include "ComputePmeCUDAMgr.h"
00065 // #endif
00066 #include "ComputeGridForceMgr.decl.h"
00067 #include "OptPmeMgr.decl.h"
00068 #include "Sync.h"
00069 #include "BackEnd.h"
00070 #include "PDB.h"
00071 #include "packmsg.h"
00072 #include "CollectionMgr.decl.h"
00073 #include "ParallelIOMgr.decl.h"
00074 #include "Vector.h"
00075 // BEGIN LA
00076 #include "Random.h"
00077 // END LA
00078 
00079 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00080 extern "C" void CApplicationInit();
00081 #endif
00082 
00083 #include "DumpBench.h"
00084 
00085 class CheckpointMsg : public CMessage_CheckpointMsg {
00086 public:
00087   int task;
00088   int replica;
00089   Controller::checkpoint checkpoint;
00090   char *key;
00091 };
00092 
00093 extern "C" {
00094   void recvCheckpointCReq_handler(envelope*);
00095   void recvCheckpointCAck_handler(envelope*);
00096 }
00097 
00098 #ifdef CMK_BALANCED_INJECTION_API
00099 #include "ckBIconfig.h"
00100 #endif
00101 
00102 #include "CollectionMgr.h"
00103 #include "CollectionMaster.h"
00104 #include "CollectionMgr.decl.h"
00105 #include "CollectionMaster.decl.h"
00106 
00107 #if USE_HPM
00108 extern "C" void HPM_Init(int);
00109 extern "C" void HPM_Start(char *label, int);
00110 extern "C" void HPM_Stop(char *label, int);
00111 extern "C" void HPM_Print(int, int);
00112 #endif
00113 
00114 #if defined(NAMD_MIC)
00115   extern void mic_dumpHostDeviceComputeMap();
00116   extern void mic_initHostDeviceLDB();
00117 #endif
00118 
00119 #ifdef MEASURE_NAMD_WITH_PAPI
00120 #include "papi.h"
00121 #if CMK_SMP
00122 #include <pthread.h>
00123 #endif
00124 #define NUM_PAPI_EVENTS 6
00125 CkpvDeclare(int *, papiEvents);
00126 
00127 #define MEASURE_PAPI_SPP 1
00128 #define MEASURE_PAPI_CACHE 0
00129 #define MEASURE_PAPI_FLOPS 0
00130 
00131 static void namdInitPapiCounters(){
00132         if(CkMyRank()==0){
00133                 //only initialize per OS process (i.e. a charm node)
00134                 int retval = PAPI_library_init(PAPI_VER_CURRENT);
00135                 if(retval != PAPI_VER_CURRENT) {
00136                         if(CkMyPe()==0){
00137                                 NAMD_die("PAPI library is not compatitible!");
00138                         }
00139                 }
00140         #if CMK_SMP
00141                 //now only consider systems that are compatible with POSIX
00142                 if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
00143                         if(CkMyPe()==0){
00144                                 NAMD_die("Multi-thread mode in PAPI could not be initialized!");
00145                         }
00146                 }
00147         #endif
00148         }
00149         CkpvInitialize(int *, papiEvents);
00150         CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS+1];
00151 
00152 #if MEASURE_PAPI_CACHE
00153         if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
00154                 CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
00155         }else{
00156                 if(CkMyPe()==0){
00157                         CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");                     
00158                 }
00159                 //if not default to PAPI_TOT_INS
00160                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00161         }
00162 
00163         if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
00164                 CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
00165         }else{
00166                 //if not default to PAPI_TOT_CYC
00167                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00168         }       
00169 #elif MEASURE_PAPI_FLOPS
00170         if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
00171                 CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
00172         }else{
00173                 if(CkMyPe()==0){
00174                         CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
00175                 }
00176                 //if not default to PAPI_TOT_INS
00177                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00178         }
00179 
00180         if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
00181                 CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
00182         }else{
00183                 //if not default to PAPI_TOT_CYC
00184                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00185         }
00186 #elif MEASURE_PAPI_SPP
00187 /* for SPP we record these
00188 1) PAPI_FP_OPS
00189 2) PAPI_TOT_INS
00190 3) perf::PERF_COUNT_HW_CACHE_LL:MISS
00191 4) DATA_PREFETCHER:ALL
00192 5) PAPI_L1_DCA
00193 6) INSTRUCTION_FETCH_STALL
00194 7) PAPI_TOT_CYC, and 
00195 8) real (wall) time
00196 */
00197         int papiEventSet = PAPI_NULL; 
00198         if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
00199           CmiAbort("PAPI failed to create event set!\n");
00200         }
00201 
00202         if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
00203                 CkpvAccess(papiEvents)[0] = PAPI_FP_OPS;
00204         }else{
00205                 if(CkMyPe()==0){
00206                         CkAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
00207                 }
00208         }
00209         if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
00210                 CkpvAccess(papiEvents)[1] = PAPI_TOT_INS;
00211         }else{
00212                 if(CkMyPe()==0){
00213                         CkAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
00214                 }
00215         }
00216         int EventCode;
00217         int ret;
00218         ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
00219         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00220           CkpvAccess(papiEvents)[2] = EventCode;
00221         }else{
00222                 if(CkMyPe()==0){
00223                         CkAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
00224                 }
00225         }
00226         ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
00227         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00228           CkpvAccess(papiEvents)[3] = EventCode;
00229         }else{
00230                 if(CkMyPe()==0){
00231                         CkAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
00232                 }
00233         }
00234         if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
00235                 CkpvAccess(papiEvents)[4] = PAPI_L1_DCA;
00236         }else{
00237                 if(CkMyPe()==0){
00238                         CkAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
00239                 }
00240         }
00241         /*      ret=PAPI_event_name_to_code("INSTRUCTION_FETCH_STALL",&EventCode);
00242         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00243           CkpvAccess(papiEvents)[5] = EventCode;
00244         }else{
00245                 if(CkMyPe()==0){
00246                         CkAbort("WARNING: INSTRUCTION_FETCH_STALL doesn't exist on this platform!");
00247                 }
00248         }
00249         */
00250         if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
00251                 CkpvAccess(papiEvents)[5] = PAPI_TOT_CYC;
00252         }else{
00253                 if(CkMyPe()==0){
00254                         CkAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
00255                 }
00256         }
00257         for(int i=0;i<NUM_PAPI_EVENTS;i++)
00258           {
00259             int papiRetValue=PAPI_add_events(papiEventSet, &CkpvAccess(papiEvents)[i],1);
00260             if (papiRetValue != PAPI_OK) {
00261               CkPrintf("failure for event %d\n",i);
00262               if (papiRetValue == PAPI_ECNFLCT) {
00263                 CmiAbort("PAPI events conflict! Please re-assign event types!\n");
00264               } else {
00265                 CmiAbort("PAPI failed to add designated events!\n");
00266               }
00267             }
00268             
00269           }
00270 #endif
00271 }
00272 #endif
00273 
00274 #ifdef OPENATOM_VERSION
00275 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
00276 {
00277   CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
00278 }
00279 #endif //OPENATOM_VERSION
00280 
00281 //======================================================================
00282 // Public Functions
00283 
00284 //----------------------------------------------------------------------
00285 
00286 int eventEndOfTimeStep;
00287 double startupTime;
00288 
00289 //----------------------------------------------------------------------
00290 // BOC constructor
00291 Node::Node(GroupInitMsg *msg)
00292 {    
00293   DebugM(4,"Creating Node\n");
00294 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00295   CApplicationInit();
00296 #endif
00297   if (CkpvAccess(Node_instance) == 0) {
00298     CkpvAccess(Node_instance) = this;
00299     eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep", 135);
00300   } else {
00301     NAMD_bug("Node::Node() - another instance of Node exists!");
00302   }
00303 
00304   CkpvAccess(BOCclass_group) = msg->group;
00305   delete msg;
00306 
00307   CkpvAccess(BOCclass_group).node = thisgroup;
00308 
00309   recvCheckpointCReq_index = CmiRegisterHandler((CmiHandler)recvCheckpointCReq_handler);
00310   recvCheckpointCAck_index = CmiRegisterHandler((CmiHandler)recvCheckpointCAck_handler);
00311 
00312   startupPhase = 0;
00313 
00314   molecule = NULL;
00315   parameters = NULL;
00316   simParameters = NULL;
00317   configList = NULL;
00318   pdb = NULL;
00319   state = NULL;
00320   output = NULL;
00321   imd = new IMDOutput;
00322   colvars = 0;
00323 
00324 #if USE_HPM
00325   // assumes that this will be done only on BG/P
00326   TopoManager *tmgr = new TopoManager();
00327   int x, y, z;
00328   tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
00329   delete tmgr;
00330 #endif
00331 
00332   specialTracing = traceAvailable() && (traceIsOn()==0);
00333 
00334   DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
00335   patchMap = PatchMap::Instance();
00336   atomMap = AtomMap::Instance();
00337   if ( CkMyRank() == 0 ) ComputeMap::Instance();
00338 
00339   //Note: Binding BOC vars such as workDistrib has been moved
00340   //to the 1st phase of startup because the in-order message delivery
00341   //is not always guaranteed --Chao Mei
00342 #ifdef CMK_BALANCED_INJECTION_API
00343   if(CkMyRank() == 0){
00344     balancedInjectionLevel=ck_get_GNI_BIConfig();
00345     // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00346     ck_set_GNI_BIConfig(20);
00347     // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00348   }
00349 #endif
00350 
00351 }
00352 
00353 //----------------------------------------------------------------------
00354 // ~Node(void) needs to clean up everything.
00355 
00356 Node::~Node(void)
00357 {
00358   delete output;
00359   delete computeMap;
00360   delete atomMap;
00361   delete patchMap;
00362   delete CkpvAccess(comm);
00363   // BEGIN LA
00364   delete rand;
00365   // END LA
00366 #ifdef MEASURE_NAMD_WITH_PAPI
00367   delete CkpvAccess(papiEvents);
00368 #endif
00369 }
00370 
00371 void Node::bindBocVars(){
00372     DebugM(4,"Binding to BOC's\n");
00373     CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00374     patchMgr = pm.ckLocalBranch();
00375     CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
00376     proxyMgr = prm.ckLocalBranch();
00377     CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
00378     workDistrib = wd.ckLocalBranch();
00379     CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00380     computeMgr = cm.ckLocalBranch();
00381     CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
00382     ldbCoordinator = lc.ckLocalBranch();
00383   #ifdef MEM_OPT_VERSION      
00384     CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
00385     ioMgr = io.ckLocalBranch();
00386   #endif
00387 
00388 }
00389 
00390 //----------------------------------------------------------------------
00391 // Malloc Test Sequence
00392 void Node::mallocTest(int step) {
00393   int MB = 1024*1024;
00394   int size = 100;
00395   char* foo = (char*) malloc(size*MB);
00396   if ( ! foo ) {
00397     char buf[256];
00398     sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
00399     NAMD_die(buf);
00400   }
00401   memset(foo,0,size*MB*sizeof(char));
00402 }
00403 
00404 void Node::mallocTestQd() {
00405   if ( mallocTest_size ) {
00406     CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
00407   } else {
00408     CkPrintf("Starting malloc test on all PEs.\n");
00409   }
00410   fflush(stdout);
00411   ++mallocTest_size;
00412   CkStartQD(CkIndex_Node::mallocTestQd(), &thishandle);
00413   (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
00414 }
00415 
00416 //----------------------------------------------------------------------
00417 // Startup Sequence
00418 
00419 void Node::messageStartUp() {
00420   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00421 }
00422 
00423 SimParameters *node_simParameters;
00424 Parameters *node_parameters;
00425 Molecule *node_molecule;
00426 
00427 extern void registerUserEventsForAllComputeObjs(void);
00428 
00429 void Node::startup() {
00430   int gotoRun = false;
00431   double newTime;
00432 
00433   if (!CkMyPe()) {
00434     if (!startupPhase) {
00435       iout << iINFO << "\n";
00436       startupTime = CmiWallTimer();
00437       iout << iINFO << "Entering startup at " << startupTime << " s, ";
00438     } else {
00439       newTime = CmiWallTimer();
00440       iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00441            << newTime - startupTime << " s, ";
00442       startupTime = newTime;
00443     }
00444     iout << memusage_MB() << " MB of memory in use\n" << endi;
00445     fflush(stdout);
00446   }
00447   switch (startupPhase) {
00448 
00449   case 0:
00450     computeMap = ComputeMap::Object();
00451     namdOneCommInit(); // Namd1.X style
00452   break;
00453 
00454   case 1:
00455       bindBocVars();
00456 
00457     // send & receive molecule, simparameters... (Namd1.X style)
00458     if (CkMyPe()) {
00459       namdOneRecv();
00460     } else {
00461       namdOneSend();
00462     }
00463   break;
00464 
00465   case 2:
00466     // fix up one-per-node objects (for SMP version)
00467     simParameters = node_simParameters;
00468     parameters = node_parameters;
00469     molecule = node_molecule;
00470 
00471     SimParameters::nonbonded_select();
00472     if ( simParameters->PMEOn ) SimParameters::pme_select();   
00473  
00474     #if !CMK_SMP || ! USE_CKLOOP
00475     //the CkLoop library should be only used in SMP mode
00476     simParameters->useCkLoop = 0;
00477     #else
00478     if ( CkNumPes() < 2 * CkNumNodes() ) simParameters->useCkLoop = 0;
00479     #endif
00480 
00481 
00482     if ( simParameters->mallocTest ) {
00483       if (!CkMyPe()) {
00484         mallocTest_size = 0;
00485         CkStartQD(CkIndex_Node::mallocTestQd(), &thishandle);
00486       }
00487       return;
00488     }
00489 
00490       
00491         #ifdef MEASURE_NAMD_WITH_PAPI
00492         if(simParameters->papiMeasure) namdInitPapiCounters();  
00493         #endif
00494     
00495     #ifdef MEM_OPT_VERSION
00496     //At this point, each Node object has received the simParameters,
00497     //parameters and the atom signatures info from the master Node
00498     //(proc 0). It's time to initialize the parallel IO manager and
00499     //read the binary per-atom file --Chao Mei
00500 
00501     //Step 1: initialize the parallel IO manager per Node
00502     ioMgr->initialize(this);
00503     #endif
00504 
00505   break;
00506 
00507   case 3:
00508 
00509     #ifdef MEM_OPT_VERSION
00510     //Step 2: read the binary per-atom files (signater index, coordinates etc.)
00511     ioMgr->readPerAtomInfo();
00512     #endif
00513 
00514   break;
00515 
00516   case 4:
00517 
00518     #ifdef MEM_OPT_VERSION
00519     //Step 3: update counters of tuples and exclusions inside Molecule object
00520     ioMgr->updateMolInfo();
00521 
00522     //Step 4: prepare distributing the atoms to neighboring procs if necessary
00523     ioMgr->migrateAtomsMGrp();
00524 
00525     //step 5: initialize patchMap and send it to every other processors
00526     //to decide atoms to patch distribution on every input processor
00527     if(!CkMyPe()) {
00528         workDistrib->patchMapInit(); // create space division
00529         workDistrib->sendPatchMap();
00530     }
00531     #endif
00532 
00533     #if USE_HPM
00534     HPM_Init(localRankOnNode);
00535     #endif    
00536 
00537     // take care of inital thread setting
00538     threadInit();
00539 
00540     // create blank AtomMap
00541     AtomMap::Object()->allocateMap(molecule->numAtoms);
00542 
00543     if (!CkMyPe()) {
00544       if (simParameters->useOptPME)
00545         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_OptPmeMgr::ckNew();
00546       else 
00547 #ifdef NAMD_CUDA
00548       if (simParameters->usePMECUDA) {
00549         // computePmeCUDAMgr was created in BackEnd.C
00550         // This empty branch is to avoid initializing ComputePmeMgr
00551       } else
00552 #endif
00553       if (simParameters->PMEOn) {
00554         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
00555       }
00556         #ifdef OPENATOM_VERSION
00557         if ( simParameters->openatomOn ) { 
00558           CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
00559         }
00560         #endif // OPENATOM_VERSION
00561 
00562     }
00563     
00564     #ifdef OPENATOM_VERSION
00565     if ( simParameters->openatomOn ) {
00566       // if ( ! CkMyPe() ) { 
00567         CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle); 
00568         startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
00569       // }
00570     }
00571     #endif // OPENATOM_VERSION
00572   
00573     // BEGIN LA
00574     rand = new Random(simParameters->randomSeed);
00575     rand->split(CkMyPe(), CkNumPes());
00576     // END LA
00577 
00578   break;
00579 
00580   case 5:
00581     #ifdef MEM_OPT_VERSION
00582     //Now, every input proc has received all the atoms necessary
00583     //to decide the patches those atoms belong to
00584     
00585     //step 1: integrate the migrated atoms into the atom list that
00586     //contains the initally distributed atoms, and sort the atoms
00587     //based on hydrogenList value
00588     ioMgr->integrateMigratedAtoms();
00589 
00590     //step 2: integrate the cluster size of each atom on each output proc
00591     ioMgr->integrateClusterSize();
00592 
00593     //step 3: calculate the number of atoms in each patch on every
00594     //input procs (atoms belonging to a patch may lie on different
00595     //procs), and reduce such info on proc 0. Such info is required
00596     //for determing which node a particular patch is assigned to.
00597     ioMgr->calcAtomsInEachPatch();
00598 
00599     //set to false to re-send PatchMap later
00600     workDistrib->setPatchMapArrived(false);
00601     #endif
00602     break;
00603   case 6:     
00604     if(simParameters->isSendSpanningTreeOn()) {                         
00605                         ProxyMgr::Object()->setSendSpanning();
00606     }
00607     if(simParameters->isRecvSpanningTreeOn()) {                         
00608                         ProxyMgr::Object()->setRecvSpanning();
00609     }
00610     if(simParameters->proxyTreeBranchFactor) {
00611                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);
00612     }
00613     #ifdef PROCTRACE_DEBUG
00614     DebugFileTrace::Instance("procTrace");
00615     #endif
00616 
00617     if (!CkMyPe()) {
00618       output = new Output; // create output object just on PE(0)
00619 
00620       #ifndef MEM_OPT_VERSION
00621       workDistrib->patchMapInit(); // create space division
00622       workDistrib->createHomePatches(); // load atoms into HomePatch(es)
00623       #endif
00624       
00625       workDistrib->assignNodeToPatch();
00626       workDistrib->mapComputes();         
00627       //ComputeMap::Object()->printComputeMap();
00628 
00629       // For MIC runs, take the additional step after the compute map has been created to
00630       //   assign the various computes to either the host or the device.  This info will
00631       //   be distributed across the PEs.
00632       #if defined(NAMD_MIC)
00633         mic_initHostDeviceLDB();
00634       #endif
00635           
00636           if(simParameters->simulateInitialMapping) {
00637           iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
00638                           << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
00639                   outputPatchComputeMaps("init_mapping", 0);
00640                   iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
00641                   BackEnd::exit();
00642           }
00643 
00644       registerUserEventsForAllComputeObjs();
00645 
00646       //in MEM_OPT_VERSION, patchMap is resent
00647       //because they have been updated since creation including
00648       //#atoms per patch, the proc a patch should stay etc. --Chao Mei
00649       workDistrib->sendPatchMap();
00650       #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00651       CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00652       //a node broadcast
00653       npm.createProxyInfo(PatchMap::Object()->numPatches());
00654       #endif
00655     }
00656     {
00657         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00658         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00659         if(CkMyRank()==0) {
00660             //just need to register once
00661             npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
00662         }
00663         npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
00664         #endif
00665     }
00666   break;
00667 
00668   case 7:
00669 #ifdef CHARM_HAS_MSA
00670     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00671       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00672       msm[CkMyPe()].initialize(new CkQdMsg);
00673     }
00674 #else
00675     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00676       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00677       MsmInitMsg *msg = new MsmInitMsg;
00678       Lattice lattice = simParameters->lattice;  // system lattice vectors
00679       ScaledPosition smin=0, smax=0;
00680       if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
00681         msg->smin = smin;
00682         msg->smax = smax;
00683         msm[CkMyPe()].initialize(msg);  // call from my own PE
00684       }
00685       else if ( ! CkMyPe() ) {
00686         pdb->get_extremes(smin, smax);  // only available on PE 0
00687         msg->smin = smin;
00688         msg->smax = smax;
00689         msm.initialize(msg);  // broadcast to chare group
00690       }
00691 
00692       /*
00693       CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00694       Node *node = nd.ckLocalBranch();
00695       ScaledPosition smin, smax;
00696       node->pdb->get_extremes(smin, smax);
00697       msg->smin = smin;                       // extreme positions in system
00698       msg->smax = smax;
00699       msm[CkMyPe()].initialize(msg);
00700       */
00701     }
00702 #endif
00703 
00704     if ( simParameters->PMEOn ) {
00705       if ( simParameters->useOptPME ) {
00706         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00707         pme[CkMyPe()].initialize(new CkQdMsg);
00708       }
00709       else {
00710         #ifdef OPENATOM_VERSION
00711         if ( simParameters->openatomOn ) { 
00712           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00713           moa[CkMyPe()].initialize(new CkQdMsg);
00714         }
00715         #endif // OPENATOM_VERSION
00716 #ifdef NAMD_CUDA
00717         if ( simParameters->usePMECUDA ) {
00718           if(CkMyRank()==0) {
00719             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00720             pme.ckLocalBranch()->initialize(new CkQdMsg);  // must run on pe 0 to call ckNew
00721           }
00722         } else 
00723 #endif
00724         {
00725           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00726           pme[CkMyPe()].initialize(new CkQdMsg);          
00727         }
00728       }
00729     }
00730     break;
00731 
00732   case 8:
00733 #ifdef NAMD_CUDA
00734     if ( (simParameters->useCUDA2 || simParameters->bondedCUDA) && CkMyRank()==0 ) {
00735       CProxy_ComputeCUDAMgr nb(CkpvAccess(BOCclass_group).computeCUDAMgr);
00736       nb.ckLocalBranch()->initialize(new CkQdMsg);
00737     }
00738 #endif
00739     break;
00740 
00741   case 9:
00742     workDistrib->sendComputeMap();
00743     break;
00744 
00745   case 10:
00746     #ifdef MEM_OPT_VERSION
00747     //migrate atoms to HomePatch processors
00748     ioMgr->sendAtomsToHomePatchProcs();
00749     #endif
00750     break;
00751     
00752   case 11:
00753     // part 2 of MSM init
00754     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00755       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00756       msm[CkMyPe()].initialize_create();  // call from my own PE
00757     }
00758 
00759     if ( simParameters->PMEOn ) {
00760       if ( simParameters->useOptPME ) {
00761         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00762         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00763       }
00764       else {
00765         #ifdef OPENATOM_VERSION
00766         if ( simParameters->openatomOn ) { 
00767           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00768           moa[CkMyPe()].initWorkers(new CkQdMsg);
00769         }
00770         #endif // OPENATOM_VERSION
00771 #ifdef NAMD_CUDA
00772         if ( simParameters->usePMECUDA ) {
00773           if(CkMyRank()==0) {
00774             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00775             pme[CkMyNode()].initialize_pencils(new CkQdMsg);
00776           }
00777         } else
00778 #endif
00779         {
00780           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00781           pme[CkMyPe()].initialize_pencils(new CkQdMsg);          
00782         }
00783       }
00784     }
00785 #ifdef CHARM_HAS_MSA
00786     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00787       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00788       msm[CkMyPe()].initWorkers(new CkQdMsg);
00789     }
00790 #else
00791     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00792       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00793       msm[CkMyPe()].update(new CkQdMsg);
00794     }
00795 #endif
00796 
00797     #ifdef MEM_OPT_VERSION
00798     //Now every processor has all the atoms it needs to create the HomePatches.
00799     //The HomePatches are created in parallel on every home patch procs.
00800     ioMgr->createHomePatches();
00801     #else
00802     if (!CkMyPe()) {
00803       workDistrib->distributeHomePatches();          
00804     }
00805     #endif
00806   break;
00807 
00808   case 12:
00809     if ( simParameters->PMEOn ) {
00810       if ( simParameters->useOptPME ) {
00811         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00812         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00813       }
00814       else {
00815         #ifdef OPENATOM_VERSION
00816         if ( simParameters->openatomOn ) { 
00817           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00818           moa[CkMyPe()].startWorkers(new CkQdMsg);
00819         }
00820         #endif // OPENATOM_VERSION
00821 #ifdef NAMD_CUDA
00822         if ( simParameters->usePMECUDA ) {
00823           if(CkMyRank()==0) {
00824             CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00825             pme[CkMyNode()].activate_pencils(new CkQdMsg);
00826           }
00827         } else
00828 #endif
00829         {
00830           CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00831           pme[CkMyPe()].activate_pencils(new CkQdMsg);          
00832         }
00833       }
00834     }
00835 #ifdef CHARM_HAS_MSA
00836     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00837       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00838       msm[CkMyPe()].startWorkers(new CkQdMsg);
00839     }
00840 #else
00841     /*
00842     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00843       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00844       //msm[CkMyPe()].startWorkers(new CkQdMsg);
00845     }
00846     */
00847 #endif
00848 
00849     proxyMgr->createProxies();  // need Home patches before this
00850     if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
00851 
00852 #ifdef NAMD_TCL
00853     // TclInitSubsystems() has a race condition so we create one interp per node here
00854     if (CkMyPe() && CkMyNodeSize() > 1 && ! CkMyRank()) Tcl_DeleteInterp(Tcl_CreateInterp());
00855 #endif
00856 
00857 #ifdef USE_NODEPATCHMGR
00858         //at this point, PatchMap info has been recved on PEs. It is time to create
00859         //the home patch spanning tree for receiving proxy list info
00860         if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
00861                 if(CkMyRank()==0) {
00862                         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00863                         npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
00864                 }
00865         }
00866 #endif
00867 
00868   break;
00869 
00870   case 13:
00871 
00872     // DMK - DEBUG - If, in MIC runs, the debug option to dump all the compute maps to files
00873     //   for debugging/verification purposes has been enabled, have each PE do so now.
00874     #if defined(NAMD_MIC)
00875       mic_dumpHostDeviceComputeMap();
00876     #endif
00877 
00878     if (!CkMyPe()) {
00879       iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
00880            << " COMPUTE OBJECTS\n" << endi;
00881     }
00882     DebugM(4,"Creating Computes\n");
00883     computeMgr->createComputes(ComputeMap::Object());
00884     DebugM(4,"Building Sequencers\n");
00885     buildSequencers();
00886     DebugM(4,"Initializing LDB\n");
00887     LdbCoordinator::Object()->initialize(PatchMap::Object(),ComputeMap::Object());
00888   break;
00889 
00890   case 14:
00891     // computes may create proxies on the fly so put these in separate phase
00892     Sync::Object()->openSync();  // decide if to open local Sync 
00893     if (proxySendSpanning || proxyRecvSpanning ) proxyMgr->buildProxySpanningTree();
00894 #ifdef CMK_BALANCED_INJECTION_API
00895     if(CkMyRank() == 0){
00896       // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00897       ck_set_GNI_BIConfig(balancedInjectionLevel);
00898       // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00899     }
00900 #endif
00901 
00902   break;
00903 
00904   case 15:
00905     {
00906         //For debugging
00907         /*if(!CkMyPe()){
00908         FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
00909         dumpbench(dumpFile);
00910         NAMD_die("Normal execution\n");
00911         }*/
00912     }
00913     #ifdef MEM_OPT_VERSION
00914     //free space in the Molecule object that are not used anymore
00915     ioMgr->freeMolSpace();
00916     #endif
00917     gotoRun = true;
00918   break;
00919 
00920   default:
00921     NAMD_bug("Startup Phase has a bug - check case statement");
00922   break;
00923 
00924   }
00925 
00926   startupPhase++;
00927   if (!CkMyPe()) {
00928     if (!gotoRun) {
00929       CkStartQD(CkCallback(CkIndex_Node::startup(), thisgroup));
00930     } else {
00931       Node::messageRun();
00932     }
00933   }
00934 }
00935 
00936 #ifdef OPENATOM_VERSION
00937 void Node::doneMoaStart()
00938 {
00939 #ifdef OPENATOM_VERSION_DEBUG
00940   CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
00941 #endif //OPENATOM_VERSION_DEBUG
00942 }
00943 #endif //OPENATOM_VERSION
00944 
00945 void Node::namdOneCommInit()
00946 {
00947   if (CkpvAccess(comm) == NULL) {
00948     CkpvAccess(comm) = new Communicate();
00949 #ifdef DPMTA
00950     pvmc_init();
00951 #endif
00952   }
00953 }
00954 
00955 // Namd 1.X style Send/Recv of simulation information
00956 
00957 void Node::namdOneRecv() {
00958   if ( CmiMyRank() ) return;
00959 
00960   MIStream *conv_msg;
00961 
00962   // Receive molecule and simulation parameter information
00963   simParameters = node_simParameters = new SimParameters;
00964   //****** BEGIN CHARMM/XPLOR type changes
00965   parameters = node_parameters = new Parameters();
00966   //****** END CHARMM/XPLOR type changes
00967   molecule = node_molecule = new Molecule(simParameters,parameters);
00968 
00969   DebugM(4, "Getting SimParameters\n");
00970   conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
00971   simParameters->receive_SimParameters(conv_msg);
00972 
00973   DebugM(4, "Getting Parameters\n");
00974   conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
00975   parameters->receive_Parameters(conv_msg);
00976 
00977   DebugM(4, "Getting Molecule\n");
00978   conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00979   // Modified by JLai -- 10.21.11
00980   molecule->receive_Molecule(conv_msg);
00981   if(simParameters->goForcesOn) {
00982     iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
00983     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00984     molecule->receive_GoMolecule(conv_msg);
00985   } 
00986   // End of modification
00987   DebugM(4, "Done Receiving\n");
00988 }
00989 
00990 void Node::namdOneSend() {
00991   node_simParameters = simParameters;
00992   node_parameters = parameters;
00993   node_molecule = molecule;
00994 
00995   MOStream *conv_msg;
00996   // I'm Pe(0) so I send what I know
00997   DebugM(4, "Sending SimParameters\n");  
00998   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
00999   simParameters->send_SimParameters(conv_msg);
01000 
01001   DebugM(4, "Sending Parameters\n");
01002   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
01003   parameters->send_Parameters(conv_msg);
01004 
01005   DebugM(4, "Sending Molecule\n");
01006   int bufSize = BUFSIZE;
01007   if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01008   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01009   // Modified by JLai -- 10.21.11
01010   molecule->send_Molecule(conv_msg);
01011   
01012   if(simParameters->goForcesOn) {
01013     iout << iINFO <<  "Master Node sending GoMolecule Information" << "\n" << endi;
01014     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01015     molecule->send_GoMolecule(conv_msg);
01016   } // End of modification
01017 }
01018 
01019 
01020 void Node::reloadStructure(const char *fname, const char *pdbname) {
01021   delete molecule;
01022   molecule = state->molecule = 0;
01023   delete pdb;
01024   pdb = state->pdb = 0;
01025   state->loadStructure(fname,pdbname,1);
01026   this->molecule = state->molecule;
01027   this->pdb = state->pdb;
01028   CProxy_Node nodeProxy(thisgroup);
01029   nodeProxy.resendMolecule();
01030 }
01031 
01032 
01033 void Node::resendMolecule() {
01034   if ( CmiMyRank() ) {
01035     return;
01036   }
01037   if ( CmiMyPe() == 0 ) {
01038     int bufSize = BUFSIZE;
01039     MOStream *conv_msg;
01040     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, bufSize);
01041     parameters->send_Parameters(conv_msg);
01042     if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01043     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01044     molecule->send_Molecule(conv_msg);
01045   } else {
01046     MIStream *conv_msg;
01047     delete parameters;
01048     parameters = new Parameters;
01049     conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
01050     parameters->receive_Parameters(conv_msg);
01051     delete molecule;
01052     molecule = new Molecule(simParameters,parameters);
01053     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
01054     molecule->receive_Molecule(conv_msg);
01055   }
01056   node_parameters = parameters;
01057   node_molecule = molecule;
01058   SimParameters::nonbonded_select();
01059   if ( simParameters->PMEOn ) SimParameters::pme_select();
01060   computeMgr->sendBuildCudaExclusions();
01061   CProxy_Node nodeProxy(thisgroup);
01062   for ( int i=0; i<CmiMyNodeSize(); ++i ) {
01063     nodeProxy[CmiMyPe()+i].resendMolecule2();
01064   }
01065 }
01066 
01067 void Node::resendMolecule2() {
01068   parameters = node_parameters;
01069   molecule = node_molecule;
01070   AtomMap::Object()->allocateMap(molecule->numAtoms);
01071 }
01072 
01073 
01074 // Initial thread setup
01075 
01076 void Node::threadInit() {
01077   // Thread initialization
01078   if (CthImplemented()) {
01079     CthSetStrategyDefault(CthSelf());
01080   } else {
01081     NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
01082   }
01083 }
01084 
01085 //
01086 void Node::buildSequencers() {
01087   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01088   ResizeArrayIter<HomePatchElem> ai(*hpl);
01089 
01090   // Controller object is only on Pe(0)
01091   if ( ! CkMyPe() ) {
01092     Controller *controller = new Controller(state);
01093     state->useController(controller);
01094   }
01095 
01096   // Assign Sequencer to all HomePatch(es)
01097   for (ai=ai.begin(); ai != ai.end(); ai++) {
01098     HomePatch *patch = (*ai).patch;
01099     Sequencer *sequencer = new Sequencer(patch);
01100     patch->useSequencer(sequencer);
01101   }
01102 }
01103 
01104 
01105 
01106 //-----------------------------------------------------------------------
01107 // Node run() - broadcast to all nodes
01108 //-----------------------------------------------------------------------
01109 void Node::messageRun() {
01110   (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
01111 }
01112 
01113 
01114 //-----------------------------------------------------------------------
01115 // run(void) runs the specified simulation for the specified number of
01116 // steps, overriding the contents of the configuration file
01117 //-----------------------------------------------------------------------
01118 void Node::run()
01119 {
01120   // Start Controller (aka scalar Sequencer) on Pe(0)
01121 //  printf("\n\n I am in Node.C in run method about to call  state->runController\n\n");
01122   if ( ! CkMyPe() ) {
01123     state->runController();
01124   }
01125 
01126   DebugM(4, "Starting Sequencers\n");
01127   // Run Sequencer on each HomePatch - i.e. start simulation
01128   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01129   ResizeArrayIter<HomePatchElem> ai(*hpl);
01130   for (ai=ai.begin(); ai != ai.end(); ai++) {
01131     HomePatch *patch = (*ai).patch;
01132 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
01133     patch->runSequencer();
01134   }
01135 
01136   if (!CkMyPe()) {
01137     double newTime = CmiWallTimer();
01138     iout << iINFO << "Startup phase " << startupPhase-1 << " took "
01139          << newTime - startupTime << " s, "
01140          << memusage_MB() << " MB of memory in use\n";
01141     iout << iINFO << "Finished startup at " << newTime << " s, "
01142          << memusage_MB() << " MB of memory in use\n\n" << endi;
01143     fflush(stdout);
01144   }
01145   
01146 }
01147 
01148 
01149 //-----------------------------------------------------------------------
01150 // Node scriptBarrier() - twiddle parameters with simulation halted
01151 //-----------------------------------------------------------------------
01152 
01153 void Node::enableScriptBarrier() {
01154   CkStartQD(CkIndex_Node::scriptBarrier(), &thishandle);
01155 }
01156 
01157 void Node::scriptBarrier() {
01158   //script->awaken();
01159 }
01160 
01161 void Node::scriptParam(ScriptParamMsg *msg) {
01162   simParameters->scriptSet(msg->param,msg->value);
01163   delete msg;
01164 }
01165 
01166 void Node::reloadCharges(const char *filename) {
01167   FILE *file = fopen(filename,"r");
01168   if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
01169 
01170   int n = molecule->numAtoms;
01171   float *charge = new float[n];
01172 
01173   for ( int i = 0; i < n; ++i ) {
01174     if ( ! fscanf(file,"%f",&charge[i]) )
01175       NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
01176   }
01177 
01178   fclose(file);
01179   CProxy_Node(thisgroup).reloadCharges(charge,n);
01180   delete [] charge;
01181 }
01182 
01183 void Node::reloadCharges(float charge[], int n) {
01184   molecule->reloadCharges(charge,n);
01185 }
01186 
01187 
01188 // BEGIN gf
01189 void Node::reloadGridforceGrid(const char * key) {
01190     DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
01191     
01192     int gridnum;
01193     MGridforceParams *mgridParams;
01194     if (key == NULL) {
01195         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01196         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01197     } else {
01198         gridnum = simParameters->mgridforcelist.index_for_key(key);
01199         mgridParams = simParameters->mgridforcelist.find_key(key);
01200     }
01201     
01202     if (gridnum < 0 || mgridParams == NULL) {
01203         NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
01204     }
01205     
01206     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01207     if (grid == NULL) {
01208         NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
01209     }
01210     grid->reinitialize(simParameters, mgridParams);
01211     
01212     CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
01213     
01214     DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
01215 }
01216 
01217 void Node::updateGridScale(const char* key, Vector scale) {
01218     DebugM(4, "updateGridScale(char*, Vector) called on node " << CkMyPe() << "\n" << endi);
01219     
01220     int gridnum;
01221     MGridforceParams* mgridParams;
01222     if (key == NULL) {
01223         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01224         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01225     } else {
01226         gridnum = simParameters->mgridforcelist.index_for_key(key);
01227         mgridParams = simParameters->mgridforcelist.find_key(key);
01228     }
01229 
01230     if (gridnum < 0 || mgridParams == NULL) {
01231         NAMD_die("Node::updateGridScale(char*, Vector): Could not find grid.");
01232     }
01233     
01234     GridforceGrid* grid = molecule->get_gridfrc_grid(gridnum);
01235     if (grid == NULL) {
01236         NAMD_bug("Node::updateGridScale(char*, Vector): grid not found");
01237     }
01238     CProxy_Node(thisgroup).updateGridScale(gridnum, scale.x, scale.y, scale.z);
01239     
01240     DebugM(4, "updateGridScale(char*, Vector) finished\n" << endi);
01241 }
01242 void Node::updateGridScale(int gridnum, float sx, float sy, float sz) {
01243     if (CmiMyRank()) return;
01244     DebugM(4, "updateGridScale(char*, int, float, float, float) called on node " << CkMyPe() << "\n" << endi);
01245        
01246     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01247     if (grid == NULL) {
01248         NAMD_bug("Node::updateGridScale(char*, int, float, float, float):grid not found");
01249     }
01250     
01251     Vector scale(sx,sy,sz);
01252     simParameters->mgridforcelist.at_index(gridnum)->gridforceScale = scale;
01253     grid->set_scale( scale );
01254 
01255     DebugM(4, "updateGridScale(char*, int, float, float, float) finished\n" << endi);
01256 }
01257 
01258 void Node::reloadGridforceGrid(int gridnum) {
01259     if (CmiMyRank()) return;
01260     DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
01261     
01262     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01263     if (grid == NULL) {
01264         NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
01265     }
01266     
01267     if (CkMyPe()) {
01268         // not node 0 -> receive grid
01269         DebugM(4, "Receiving grid\n");
01270         
01271         delete grid;
01272         
01273         MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
01274         grid = GridforceGrid::unpack_grid(gridnum, msg);
01275         molecule->set_gridfrc_grid(gridnum, grid);
01276         delete msg;
01277     } else {
01278         // node 0 -> send grid
01279         DebugM(4, "Sending grid\n");
01280         
01281         MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
01282         GridforceGrid::pack_grid(grid, msg);
01283         msg->end();
01284         delete msg;
01285     }
01286     
01287     DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
01288 }
01289 // END gf
01290 
01291 
01292 // initiating replica
01293 void Node::sendCheckpointReq(int remote, const char *key, int task, Lattice &lat, ControllerState &cs) {
01294   CheckpointMsg *msg = new (1+strlen(key),0) CheckpointMsg;
01295   msg->replica = CmiMyPartition();
01296   msg->task = task;
01297   msg->checkpoint.lattice = lat;
01298   msg->checkpoint.state = cs;
01299   strcpy(msg->key,key);
01300   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01301   CmiSetHandler(env,recvCheckpointCReq_index);
01302 #if CMK_HAS_PARTITION
01303   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01304 #else
01305   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01306 #endif
01307 }
01308 
01309 // responding replica
01310 extern "C" {
01311   void recvCheckpointCReq_handler(envelope *env) {
01312     Node::Object()->recvCheckpointReq(CheckpointMsg::unpack(EnvToUsr(env)));
01313   }
01314 }
01315 
01316 // responding replica
01317 void Node::recvCheckpointReq(CheckpointMsg *msg) {
01318   state->controller->recvCheckpointReq(msg->key,msg->task,msg->checkpoint);
01319 
01320   int remote = msg->replica;
01321   msg->replica = CmiMyPartition();
01322   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01323   CmiSetHandler(env,recvCheckpointCAck_index);
01324 #if CMK_HAS_PARTITION
01325   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01326 #else
01327   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01328 #endif
01329 }
01330 
01331 // initiating replica
01332 extern "C" {
01333   void recvCheckpointCAck_handler(envelope *env) {
01334     Node::Object()->recvCheckpointAck(CheckpointMsg::unpack(EnvToUsr(env)));
01335   }
01336 }
01337 
01338 // initiating replica
01339 void Node::recvCheckpointAck(CheckpointMsg *msg) {
01340   state->controller->recvCheckpointAck(msg->checkpoint);
01341   delete msg;
01342 }
01343 
01344 
01345 void Node::sendEnableExitScheduler(void) {
01346   //CmiPrintf("sendEnableExitScheduler\n");
01347   CProxy_Node nodeProxy(thisgroup);
01348   nodeProxy[0].recvEnableExitScheduler();
01349 }
01350 
01351 void Node::recvEnableExitScheduler(void) {
01352   //CmiPrintf("recvEnableExitScheduler\n");
01353   enableExitScheduler();
01354 }
01355 
01356 void Node::enableExitScheduler(void) {
01357   if ( CkMyPe() ) {
01358     sendEnableExitScheduler();
01359   } else {
01360     CkStartQD(CkIndex_Node::exitScheduler(), &thishandle);
01361   }
01362 }
01363 
01364 void Node::exitScheduler(void) {
01365   //CmiPrintf("exitScheduler %d\n",CkMyPe());
01366   CsdExitScheduler();
01367 }
01368 
01369 void Node::sendEnableEarlyExit(void) {
01370   CProxy_Node nodeProxy(thisgroup);
01371   nodeProxy[0].recvEnableEarlyExit();
01372 }
01373 
01374 void Node::recvEnableEarlyExit() {
01375   enableEarlyExit();
01376 }
01377 
01378 void Node::enableEarlyExit(void) {
01379   if ( CkMyPe() ) {
01380     sendEnableEarlyExit();
01381   } else {
01382     CkStartQD(CkIndex_Node::earlyExit(),&thishandle);
01383   }
01384 }
01385 
01386 void Node::earlyExit(void) {
01387   iout << iERROR << "Exiting prematurely; see error messages above.\n" << endi;
01388   if ( CmiNumPartitions() > 1 ) NAMD_quit("Exiting prematurely; see error messages above.");
01389   BackEnd::exit();
01390 }
01391 
01392 
01393 //------------------------------------------------------------------------
01394 // Some odd utilities
01395 //------------------------------------------------------------------------
01396 void Node::saveMolDataPointers(NamdState *state)
01397 {
01398   this->molecule = state->molecule;
01399   this->parameters = state->parameters;
01400   this->simParameters = state->simParameters;
01401   this->configList = state->configList;
01402   this->pdb = state->pdb;
01403   this->state = state;
01404 }
01405 
01406 // entry methods for BG/P HPM (performance counters) library
01407 void Node::startHPM() {
01408 #if USE_HPM
01409   HPM_Start("500 steps", localRankOnNode);
01410 #endif
01411 }
01412 
01413 void Node::stopHPM() {
01414 #if USE_HPM
01415   HPM_Stop("500 steps", localRankOnNode);
01416   HPM_Print(CkMyPe(), localRankOnNode);
01417 #endif
01418 }
01419 
01420 void Node::traceBarrier(int turnOnTrace, int step){
01421         curTimeStep = step;
01422         if(turnOnTrace) traceBegin();
01423         else traceEnd();
01424 
01425     if(turnOnTrace) CmiTurnOnStats();
01426     else CmiTurnOffStats();
01427 
01428         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());    
01429         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01430         CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
01431         contribute(0, NULL, CkReduction::sum_int, cb);
01432         
01433 }
01434 
01435 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
01436         CmiAssert(CmiMyPe()==0);
01437         delete msg;     
01438         state->controller->resumeAfterTraceBarrier(curTimeStep);
01439 }
01440 
01441 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
01442 #ifdef MEASURE_NAMD_WITH_PAPI
01443         curMFlopStep = step;
01444         double results[NUM_PAPI_EVENTS+1];
01445 
01446         if(turnOnMeasure){              
01447           CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]=CmiWallTimer();
01448 
01449           long long counters[NUM_PAPI_EVENTS+1];
01450           int ret=PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
01451           if(ret==PAPI_OK)
01452             {
01453               //              CkPrintf("traceBarrier start counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01454             }
01455           else
01456             {
01457               CkPrintf("error PAPI_start_counters (%d) at step %d called on proc %d\n",ret , step, CkMyPe());
01458             }
01459           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)!=PAPI_OK)
01460             {
01461               CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01462             };
01463         }else{
01464           long long counters[NUM_PAPI_EVENTS+1];
01465           for(int i=0;i<NUM_PAPI_EVENTS;i++)  counters[i]=0LL;
01466           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)==PAPI_OK)
01467             {
01468 #if !MEASURE_PAPI_SPP
01469               results[0] = (double)counters[0]/1e6;
01470               results[1] = (double)counters[1]/1e6;
01471 #else
01472               for(int i=0;i<NUM_PAPI_EVENTS;i++)  results[i] = counters[i]/1e6;
01473 #endif
01474               //              for(int i=0;i<NUM_PAPI_EVENTS;i++) CkPrintf("[%d] counter %d is %ld\n",CkMyPe(),i,counters[i]);
01475             }
01476           else
01477             {
01478               //              CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01479             }
01480           //      CkPrintf("traceBarrier stop counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01481                 
01482           PAPI_stop_counters(counters, NUM_PAPI_EVENTS);        
01483         }
01484         if(CkMyPe()==0)
01485           //        CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01486         results[NUM_PAPI_EVENTS]=CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]; //starttime
01487         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01488         CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
01489         contribute(sizeof(double)*(NUM_PAPI_EVENTS+1), &results, CkReduction::sum_double, cb);  
01490 #endif
01491 }
01492 
01493 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
01494 #ifdef MEASURE_NAMD_WITH_PAPI
01495   
01496         if(simParameters->papiMeasureStartStep != curMFlopStep) {
01497                 double *results = (double *)msg->getData();
01498                 double endtime=CmiWallTimer();
01499                 int bstep = simParameters->papiMeasureStartStep;
01500                 int estep = bstep + simParameters->numPapiMeasureSteps;
01501 #if MEASURE_PAPI_SPP
01502                 CkPrintf("SPP INFO: PAPI_FP_OPS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[0]);
01503                 CkPrintf("SPP INFO: PAPI_TOT_INS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[1]);
01504                 CkPrintf("SPP INFO: perf::PERF_COUNT_HW_CACHE_LL:MISS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[2]);
01505                 CkPrintf("SPP INFO: DATA_PREFETCHER:ALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[3]);
01506                 CkPrintf("SPP INFO: PAPI_L1_DCA timestep %d to %d is %lf(1e6)\n", bstep,estep,results[4]);
01507                 CkPrintf("SPP INFO: PAPI_TOT_CYC timestep %d to % is %lf(1e6)\n", bstep,estep,results[5]);
01508                 //              CkPrintf("SPP INFO: INSTRUCTION_FETCH_STALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[6]);
01509                 //              CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]/CkNumPes());
01510                 CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]);
01511                 CkPrintf("SPP INFO: endtime %lf avgtime %lf tottime %lf\n", endtime,results[NUM_PAPI_EVENTS]/CkNumPes(),results[NUM_PAPI_EVENTS] );
01512 #else
01513                 if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
01514                         double totalFPIns = results[0];
01515                         if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
01516                         CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n", 
01517                                          bstep, estep, totalFPIns/CkNumPes());
01518                 }else{
01519                         char nameBuf[PAPI_MAX_STR_LEN];
01520                         CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ", 
01521                                          bstep, estep);
01522                         for(int i=0; i<NUM_PAPI_EVENTS; i++) {
01523                                 PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
01524                                 CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
01525                         }
01526                         CkPrintf("per processor\n");
01527                 }               
01528 #endif
01529         }
01530         delete msg;     
01531         state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
01532 #endif
01533 }
01534 
01535 extern char *gNAMDBinaryName;
01536 void Node::outputPatchComputeMaps(const char *filename, int tag){
01537         if(!simParameters->outputMaps && !simParameters->simulateInitialMapping) return;
01538 
01539         int numpes = CkNumPes();
01540         int nodesize = CkMyNodeSize();
01541         if(simParameters->simulateInitialMapping) {
01542                 numpes = simParameters->simulatedPEs;
01543                 nodesize = simParameters->simulatedNodeSize;
01544         }
01545 
01546         char fname[128];
01547         sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
01548 
01549         FILE *fp = fopen(fname, "w");
01550         if(fp == NULL) {
01551                 NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
01552                 return;
01553         }
01554         PatchMap *pMap = PatchMap::Object();
01555         ComputeMap *cMap = ComputeMap::Object();
01556         int numPatches = pMap->numPatches();
01557         int numComputes = cMap->numComputes();
01558         fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes, 
01559                         pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
01560         //output PatchMap info
01561         for(int i=0; i<numPatches; i++) {
01562         #ifdef MEM_OPT_VERSION
01563                 fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
01564         #else
01565                 fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
01566         #endif
01567         }
01568 
01569         //output ComputeMap info
01570         for(int i=0; i<numComputes; i++) {              
01571                 fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));             
01572         }
01573 }
01574 
01575 
01576 //======================================================================
01577 // Private functions
01578 
01579 #include "Node.def.h"
01580 

Generated on Tue Sep 25 01:17:14 2018 for NAMD by  doxygen 1.4.7