Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | File List | Class Members | File Members

Node.C

Go to the documentation of this file.
00001 
00007 /*
00008    Toplevel routines for initializing a Node for a simulation
00009    one Node per Pe (processor element).
00010 */
00011 
00012 #if !defined(WIN32) || defined(__CYGWIN__)
00013 #include <unistd.h>
00014 #endif
00015 #include "InfoStream.h"
00016 #include "Node.decl.h"
00017 #include "Node.h"
00018 #ifdef DPMTA
00019 #include <pvm3.h>
00020 #endif
00021 
00022 #include "ProcessorPrivate.h"
00023 
00024 #define MIN_DEBUG_LEVEL 3
00025 //#define DEBUGM
00026 #include "Debug.h"
00027 
00028 #include <stdio.h>
00029 #include <converse.h>
00030 #include "memusage.h"
00031 #include "IMDOutput.h"
00032 #include "Lattice.h"
00033 #include "ComputeMsmMsa.h"  // needed for MsmMsaData definition
00034 #include "ComputeMsm.h"     // needed for MsmInitMsg definition
00035 #include "main.decl.h"
00036 #include "main.h"
00037 #include "WorkDistrib.h"
00038 #include "PatchMgr.h"
00039 #include "Patch.h"
00040 #include "Compute.h"
00041 #include "ComputeMap.h"
00042 #include "ComputeMgr.h"
00043 #include "Molecule.h"
00044 #include "HomePatchList.h"
00045 #include "AtomMap.h"
00046 #include "Sequencer.h"
00047 #include "Controller.h"
00048 #include "NamdState.h"
00049 #include "Output.h"
00050 #include "ProxyMgr.h"
00051 #include "PatchMap.h"
00052 #include "PatchMap.inl"
00053 #include "Parameters.h"
00054 #include "SimParameters.h"
00055 #include "Communicate.h"
00056 #include "LdbCoordinator.h"
00057 #include "ScriptTcl.h"
00058 #include "ComputeMgr.decl.h"
00059 #include "ComputePmeMgr.decl.h"
00060 #include "ComputeGridForceMgr.decl.h"
00061 #include "OptPmeMgr.decl.h"
00062 #include "Sync.h"
00063 #include "BackEnd.h"
00064 #include "PDB.h"
00065 #include "packmsg.h"
00066 #include "CollectionMgr.decl.h"
00067 #include "ParallelIOMgr.decl.h"
00068 // BEGIN LA
00069 #include "Random.h"
00070 // END LA
00071 
00072 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00073 extern "C" void CApplicationInit();
00074 #endif
00075 
00076 #include "DumpBench.h"
00077 
00078 #ifdef MEM_OPT_VERSION
00079 #include "CollectionMgr.h"
00080 #include "CollectionMaster.h"
00081 #include "CollectionMgr.decl.h"
00082 #include "CollectionMaster.decl.h"
00083 #endif
00084 
00085 #if USE_HPM
00086 extern "C" void HPM_Init(int);
00087 extern "C" void HPM_Start(char *label, int);
00088 extern "C" void HPM_Stop(char *label, int);
00089 extern "C" void HPM_Print(int, int);
00090 #endif
00091 
00092 #ifdef MEASURE_NAMD_WITH_PAPI
00093 #include "papi.h"
00094 #if CMK_SMP
00095 #include <pthread.h>
00096 #endif
00097 #define NUM_PAPI_EVENTS 2
00098 CkpvDeclare(int *, papiEvents);
00099 
00100 #define MEASURE_PAPI_CACHE 1
00101 #define MEASURE_PAPI_FLOPS 0
00102 
00103 static void namdInitPapiCounters(){
00104         if(CkMyRank()==0){
00105                 //only initialize per OS process (i.e. a charm node)
00106                 int retval = PAPI_library_init(PAPI_VER_CURRENT);
00107                 if(retval != PAPI_VER_CURRENT) {
00108                         if(CkMyPe()==0){
00109                                 CkPrintf("ERROR: PAPI library is not compatitible!");
00110                                 CkExit();
00111                         }
00112                 }
00113         #if CMK_SMP
00114                 //now only consider systems that are compatible with POSIX
00115                 if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
00116                         if(CkMyPe()==0){
00117                                 CkPrintf("ERROR: multi-thread mode in PAPI could not be initialized!");
00118                                 CkExit();
00119                         }
00120                 }
00121         #endif
00122         }
00123         CkpvInitialize(int *, papiEvents);
00124         CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS];
00125 
00126 #if MEASURE_PAPI_CACHE
00127         if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
00128                 CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
00129         }else{
00130                 if(CkMyPe()==0){
00131                         CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");                     
00132                 }
00133                 //if not default to PAPI_TOT_INS
00134                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00135         }
00136 
00137         if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
00138                 CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
00139         }else{
00140                 //if not default to PAPI_TOT_CYC
00141                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00142         }       
00143 #elif MEASURE_PAPI_FLOPS
00144         if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
00145                 CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
00146         }else{
00147                 if(CkMyPe()==0){
00148                         CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
00149                 }
00150                 //if not default to PAPI_TOT_INS
00151                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00152         }
00153 
00154         if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
00155                 CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
00156         }else{
00157                 //if not default to PAPI_TOT_CYC
00158                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00159         }
00160 #endif
00161 }
00162 #endif
00163 
00164 #ifdef OPENATOM_VERSION
00165 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
00166 {
00167   CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
00168 }
00169 #endif //OPENATOM_VERSION
00170 
00171 //======================================================================
00172 // Public Functions
00173 
00174 //----------------------------------------------------------------------
00175 
00176 int eventEndOfTimeStep;
00177 double startupTime;
00178 
00179 //----------------------------------------------------------------------
00180 // BOC constructor
00181 Node::Node(GroupInitMsg *msg)
00182 {    
00183   DebugM(4,"Creating Node\n");
00184 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00185   CApplicationInit();
00186 #endif
00187   if (CkpvAccess(Node_instance) == 0) {
00188     CkpvAccess(Node_instance) = this;
00189     eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep");
00190   } else {
00191     NAMD_bug("Node::Node() - another instance of Node exists!");
00192   }
00193 
00194   CkpvAccess(BOCclass_group) = msg->group;
00195   delete msg;
00196 
00197   CkpvAccess(BOCclass_group).node = thisgroup;
00198 
00199   startupPhase = 0;
00200 
00201   molecule = NULL;
00202   parameters = NULL;
00203   simParameters = NULL;
00204   configList = NULL;
00205   pdb = NULL;
00206   state = NULL;
00207   output = NULL;
00208   imd = new IMDOutput;
00209   colvars = 0;
00210 
00211 #if USE_HPM
00212   // assumes that this will be done only on BG/P
00213   TopoManager *tmgr = new TopoManager();
00214   int x, y, z;
00215   tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
00216   delete tmgr;
00217 #endif
00218 
00219   specialTracing = traceAvailable() && (traceIsOn()==0);
00220 
00221   DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
00222   patchMap = PatchMap::Instance();
00223   atomMap = AtomMap::Instance();
00224   if ( CkMyRank() == 0 ) ComputeMap::Instance();
00225 
00226   //Note: Binding BOC vars such as workDistrib has been moved
00227   //to the 1st phase of startup because the in-order message delivery
00228   //is not always guaranteed --Chao Mei
00229 }
00230 
00231 //----------------------------------------------------------------------
00232 // ~Node(void) needs to clean up everything.
00233 
00234 Node::~Node(void)
00235 {
00236   delete output;
00237   delete computeMap;
00238   delete atomMap;
00239   delete patchMap;
00240   delete CkpvAccess(comm);
00241   // BEGIN LA
00242   delete rand;
00243   // END LA
00244 #ifdef MEASURE_NAMD_WITH_PAPI
00245   delete CkpvAccess(papiEvents);
00246 #endif
00247 }
00248 
00249 void Node::bindBocVars(){
00250     DebugM(4,"Binding to BOC's\n");
00251     CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00252     patchMgr = pm.ckLocalBranch();
00253     CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
00254     proxyMgr = prm.ckLocalBranch();
00255     CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
00256     workDistrib = wd.ckLocalBranch();
00257     CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00258     computeMgr = cm.ckLocalBranch();
00259     CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
00260     ldbCoordinator = lc.ckLocalBranch();
00261   #ifdef MEM_OPT_VERSION      
00262     CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
00263     ioMgr = io.ckLocalBranch();
00264   #endif
00265 }
00266 
00267 //----------------------------------------------------------------------
00268 // Malloc Test Sequence
00269 void Node::mallocTest(int step) {
00270   int MB = 1024*1024;
00271   int size = 100;
00272   char* foo = (char*) malloc(size*MB);
00273   if ( ! foo ) {
00274     char buf[256];
00275     sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
00276     NAMD_die(buf);
00277   }
00278   memset(foo,0,size*MB*sizeof(char));
00279 }
00280 
00281 void Node::mallocTestQd(CkQdMsg *qmsg) {
00282   delete qmsg;
00283   if ( mallocTest_size ) {
00284     CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
00285   } else {
00286     CkPrintf("Starting malloc test on all PEs.\n");
00287   }
00288   fflush(stdout);
00289   ++mallocTest_size;
00290   CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00291   (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
00292 }
00293 
00294 //----------------------------------------------------------------------
00295 // Startup Sequence
00296 
00297 void Node::messageStartUp() {
00298   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00299 }
00300 
00301 void Node::startUp(CkQdMsg *qmsg) {
00302   delete qmsg;
00303   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00304 }
00305 
00306 SimParameters *node_simParameters;
00307 Parameters *node_parameters;
00308 Molecule *node_molecule;
00309 
00310 extern void registerUserEventsForAllComputeObjs(void);
00311 
00312 void Node::startup() {
00313   int gotoRun = false;
00314   double newTime;
00315 
00316   if (!CkMyPe()) {
00317     if (!startupPhase) {
00318       iout << iINFO << "\n";
00319       startupTime = CmiWallTimer();
00320       iout << iINFO << "Entering startup at " << startupTime << " s, ";
00321     } else {
00322       newTime = CmiWallTimer();
00323       iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00324            << newTime - startupTime << " s, ";
00325       startupTime = newTime;
00326     }
00327     iout << memusage_MB() << " MB of memory in use\n" << endi;
00328     fflush(stdout);
00329   }
00330   
00331   switch (startupPhase) {
00332 
00333   case 0:
00334     computeMap = ComputeMap::Object();
00335 
00336     namdOneCommInit(); // Namd1.X style
00337   break;
00338 
00339   case 1:
00340       bindBocVars();
00341 
00342     // send & receive molecule, simparameters... (Namd1.X style)
00343     if (CkMyPe()) {
00344       namdOneRecv();
00345     } else {
00346       namdOneSend();
00347     }
00348   break;
00349 
00350   case 2:
00351     // fix up one-per-node objects (for SMP version)
00352     simParameters = node_simParameters;
00353     parameters = node_parameters;
00354     molecule = node_molecule;
00355     
00356     #if !CMK_SMP || ! USE_NODEHELPER
00357     //the NodeHelper library should be only used in SMP mode
00358     simParameters->useNodeHelper = 0;
00359     #endif
00360 
00361 
00362     if ( simParameters->mallocTest ) {
00363       if (!CkMyPe()) {
00364         mallocTest_size = 0;
00365         CkStartQD(CkIndex_Node::mallocTestQd((CkQdMsg*)0),&thishandle);
00366       }
00367       return;
00368     }
00369 
00370       
00371         #ifdef MEASURE_NAMD_WITH_PAPI
00372         if(simParameters->papiMeasure) namdInitPapiCounters();  
00373         #endif
00374     
00375     #ifdef MEM_OPT_VERSION
00376     //At this point, each Node object has received the simParameters,
00377     //parameters and the atom signatures info from the master Node
00378     //(proc 0). It's time to initialize the parallel IO manager and
00379     //read the binary per-atom file --Chao Mei
00380 
00381     //Step 1: initialize the parallel IO manager per Node
00382     ioMgr->initialize(this);
00383 
00384     //Step 2: read the binary per-atom files (signater index, coordinates etc.)
00385     ioMgr->readPerAtomInfo();
00386 
00387     //Step 3: update counters of tuples and exclusions inside Molecule object
00388     ioMgr->updateMolInfo();
00389 
00390     //Step 4: prepare distributing the atoms to neighboring procs if necessary
00391     ioMgr->migrateAtomsMGrp();
00392 
00393     //step 5: initialize patchMap and send it to every other processors
00394     //to decide atoms to patch distribution on every input processor
00395     if(!CkMyPe()) {
00396         workDistrib->patchMapInit(); // create space division
00397         workDistrib->sendPatchMap();
00398     }
00399     #endif
00400 
00401     #if USE_HPM
00402     HPM_Init(localRankOnNode);
00403     #endif    
00404 
00405     // take care of inital thread setting
00406     threadInit();
00407 
00408     // create blank AtomMap
00409     AtomMap::Object()->allocateMap(molecule->numAtoms);
00410 
00411     if (!CkMyPe()) {
00412       if (simParameters->useOptPME)
00413         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_OptPmeMgr::ckNew();
00414       else 
00415         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
00416         #ifdef OPENATOM_VERSION
00417         if ( simParameters->openatomOn ) { 
00418           CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
00419         }
00420         #endif // OPENATOM_VERSION
00421     }
00422     
00423     #ifdef OPENATOM_VERSION
00424     if ( simParameters->openatomOn ) {
00425       // if ( ! CkMyPe() ) { 
00426         CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle); 
00427         startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
00428       // }
00429     }
00430     #endif // OPENATOM_VERSION
00431   
00432     // BEGIN LA
00433     rand = new Random(simParameters->randomSeed);
00434     rand->split(CkMyPe(), CkNumPes());
00435     // END LA
00436 
00437   break;
00438 
00439   case 3:
00440     #ifdef MEM_OPT_VERSION
00441     //Now, every input proc has received all the atoms necessary
00442     //to decide the patches those atoms belong to
00443     
00444     //step 1: integrate the migrated atoms into the atom list that
00445     //contains the initally distributed atoms, and sort the atoms
00446     //based on hydrogenList value
00447     ioMgr->integrateMigratedAtoms();
00448 
00449     //step 2: integrate the cluster size of each atom on each output proc
00450     ioMgr->integrateClusterSize();
00451 
00452     //step 3: calculate the number of atoms in each patch on every
00453     //input procs (atoms belonging to a patch may lie on different
00454     //procs), and reduce such info on proc 0. Such info is required
00455     //for determing which node a particular patch is assigned to.
00456     ioMgr->calcAtomsInEachPatch();
00457 
00458     //set to false to re-send PatchMap later
00459     workDistrib->setPatchMapArrived(false);
00460     #endif
00461     break;
00462   case 4:     
00463     if(simParameters->isSendSpanningTreeOn()) {                         
00464                         ProxyMgr::Object()->setSendSpanning();
00465                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);                             
00466     }
00467     if(simParameters->isRecvSpanningTreeOn()) {                         
00468                         ProxyMgr::Object()->setRecvSpanning();
00469                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);                             
00470     }
00471     #ifdef PROCTRACE_DEBUG
00472     DebugFileTrace::Instance("procTrace");
00473     #endif
00474 
00475     if (!CkMyPe()) {
00476       output = new Output; // create output object just on PE(0)
00477 
00478       #ifndef MEM_OPT_VERSION
00479       workDistrib->patchMapInit(); // create space division
00480       workDistrib->createHomePatches(); // load atoms into HomePatch(es)
00481       #endif
00482       
00483       workDistrib->assignNodeToPatch();
00484       workDistrib->mapComputes();         
00485       //ComputeMap::Object()->printComputeMap();
00486           
00487           if(simParameters->simulateInitialMapping) {
00488           iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
00489                           << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
00490                   outputPatchComputeMaps("init_mapping", 0);
00491                   iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
00492                   CkExit();
00493           }
00494 
00495       registerUserEventsForAllComputeObjs();
00496 
00497       //in MEM_OPT_VERSION, patchMap is resent
00498       //because they have been updated since creation including
00499       //#atoms per patch, the proc a patch should stay etc. --Chao Mei
00500       workDistrib->sendPatchMap();
00501       #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00502       CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00503       //a node broadcast
00504       npm.createProxyInfo(PatchMap::Object()->numPatches());
00505       #endif
00506     }
00507     {
00508         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00509         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00510         if(CkMyRank()==0) {
00511             //just need to register once
00512             npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
00513         }
00514         npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
00515         #endif
00516     }
00517   break;
00518 
00519   case 5:
00520     if ( simParameters->PMEOn ) {
00521       if ( simParameters->useOptPME ) {
00522         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00523         pme[CkMyPe()].initialize(new CkQdMsg);
00524       }
00525       else {
00526         #ifdef OPENATOM_VERSION
00527         if ( simParameters->openatomOn ) { 
00528           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00529           moa[CkMyPe()].initialize(new CkQdMsg);
00530         }
00531         #endif // OPENATOM_VERSION
00532         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00533         pme[CkMyPe()].initialize(new CkQdMsg);
00534       }
00535     }
00536 #ifdef CHARM_HAS_MSA
00537     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00538       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00539       msm[CkMyPe()].initialize(new CkQdMsg);
00540     }
00541 #else
00542     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00543       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00544       MsmInitMsg *msg = new MsmInitMsg;
00545       Lattice lattice = simParameters->lattice;  // system lattice vectors
00546       ScaledPosition smin=0, smax=0;
00547       if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
00548         msg->smin = smin;
00549         msg->smax = smax;
00550         msm[CkMyPe()].initialize(msg);  // call from my own PE
00551       }
00552       else if ( ! CkMyPe() ) {
00553         pdb->get_extremes(smin, smax);  // only available on PE 0
00554         msg->smin = smin;
00555         msg->smax = smax;
00556         msm.initialize(msg);  // broadcast to chare group
00557       }
00558 
00559       /*
00560       CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00561       Node *node = nd.ckLocalBranch();
00562       ScaledPosition smin, smax;
00563       node->pdb->get_extremes(smin, smax);
00564       msg->smin = smin;                       // extreme positions in system
00565       msg->smax = smax;
00566       msm[CkMyPe()].initialize(msg);
00567       */
00568     }
00569 #endif
00570 
00571     if (!CkMyPe()) {
00572       workDistrib->sendComputeMap();
00573     }
00574 
00575     #ifdef MEM_OPT_VERSION
00576     //migrate atoms to HomePatch processors
00577     ioMgr->sendAtomsToHomePatchProcs();
00578     #endif
00579     break;
00580     
00581   case 6:
00582     if ( simParameters->PMEOn ) {
00583       if ( simParameters->useOptPME ) {
00584         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00585         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00586       }
00587       else {
00588         #ifdef OPENATOM_VERSION
00589         if ( simParameters->openatomOn ) { 
00590           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00591           moa[CkMyPe()].initWorkers(new CkQdMsg);
00592         }
00593         #endif // OPENATOM_VERSION
00594         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00595         pme[CkMyPe()].initialize_pencils(new CkQdMsg);
00596       }
00597     }
00598 #ifdef CHARM_HAS_MSA
00599     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00600       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00601       msm[CkMyPe()].initWorkers(new CkQdMsg);
00602     }
00603 #else
00604     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00605       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00606       msm[CkMyPe()].update(new CkQdMsg);
00607     }
00608 #endif
00609 
00610     #ifdef MEM_OPT_VERSION
00611     //Now every processor has all the atoms it needs to create the HomePatches.
00612     //The HomePatches are created in parallel on every home patch procs.
00613     ioMgr->createHomePatches();
00614     #else
00615     if (!CkMyPe()) {
00616       workDistrib->distributeHomePatches();          
00617     }
00618     #endif
00619   break;
00620 
00621   case 7:
00622     if ( simParameters->PMEOn ) {
00623       if ( simParameters->useOptPME ) {
00624         CProxy_OptPmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00625         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00626       }
00627       else {
00628         #ifdef OPENATOM_VERSION
00629         if ( simParameters->openatomOn ) { 
00630           CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00631           moa[CkMyPe()].startWorkers(new CkQdMsg);
00632         }
00633         #endif // OPENATOM_VERSION
00634         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00635         pme[CkMyPe()].activate_pencils(new CkQdMsg);
00636       }
00637     }
00638 #ifdef CHARM_HAS_MSA
00639     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00640       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00641       msm[CkMyPe()].startWorkers(new CkQdMsg);
00642     }
00643 #else
00644     /*
00645     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00646       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00647       //msm[CkMyPe()].startWorkers(new CkQdMsg);
00648     }
00649     */
00650 #endif
00651 
00652     proxyMgr->createProxies();  // need Home patches before this
00653     if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
00654 
00655 #ifdef USE_NODEPATCHMGR
00656         //at this point, PatchMap info has been recved on PEs. It is time to create
00657         //the home patch spanning tree for receiving proxy list info
00658         if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
00659                 if(CkMyRank()==0) {
00660                         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00661                         npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
00662                 }
00663         }
00664 #endif
00665 
00666   break;
00667 
00668   case 8:
00669     if (!CkMyPe()) {
00670       iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
00671            << " COMPUTE OBJECTS\n" << endi;
00672     }
00673     DebugM(4,"Creating Computes\n");
00674     computeMgr->createComputes(ComputeMap::Object());
00675     DebugM(4,"Building Sequencers\n");
00676     buildSequencers();
00677     DebugM(4,"Initializing LDB\n");
00678     LdbCoordinator::Object()->initialize(PatchMap::Object(),ComputeMap::Object());
00679   break;
00680 
00681   case 9:
00682     // computes may create proxies on the fly so put these in separate phase
00683     Sync::Object()->openSync();  // decide if to open local Sync 
00684     if (proxySendSpanning || proxyRecvSpanning ) proxyMgr->buildProxySpanningTree();
00685   break;
00686 
00687   case 10:
00688     {
00689         //For debugging
00690         /*if(!CkMyPe()){
00691         FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
00692         dumpbench(dumpFile);
00693         NAMD_die("Normal execution\n");
00694         }*/
00695     }
00696     #ifdef MEM_OPT_VERSION
00697     //free space in the Molecule object that are not used anymore
00698     ioMgr->freeMolSpace();
00699     #endif
00700     gotoRun = true;
00701   break;
00702 
00703   default:
00704     NAMD_bug("Startup Phase has a bug - check case statement");
00705   break;
00706 
00707   }
00708 
00709   startupPhase++;
00710   if (!CkMyPe()) {
00711     if (!gotoRun) {
00712       CkStartQD(CkIndex_Node::startUp((CkQdMsg*)0),&thishandle);
00713     } else {
00714       Node::messageRun();
00715     }
00716   }
00717 }
00718 
00719 #ifdef OPENATOM_VERSION
00720 void Node::doneMoaStart()
00721 {
00722 #ifdef OPENATOM_VERSION_DEBUG
00723   CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
00724 #endif //OPENATOM_VERSION_DEBUG
00725 }
00726 #endif //OPENATOM_VERSION
00727 
00728 void Node::namdOneCommInit()
00729 {
00730   if (CkpvAccess(comm) == NULL) {
00731     CkpvAccess(comm) = new Communicate();
00732 #ifdef DPMTA
00733     pvmc_init();
00734 #endif
00735   }
00736 }
00737 
00738 // Namd 1.X style Send/Recv of simulation information
00739 
00740 void Node::namdOneRecv() {
00741   if ( CmiMyRank() ) return;
00742 
00743   MIStream *conv_msg;
00744 
00745   // Receive molecule and simulation parameter information
00746   simParameters = node_simParameters = new SimParameters;
00747   //****** BEGIN CHARMM/XPLOR type changes
00748   parameters = node_parameters = new Parameters();
00749   //****** END CHARMM/XPLOR type changes
00750   molecule = node_molecule = new Molecule(simParameters,parameters);
00751 
00752   DebugM(4, "Getting SimParameters\n");
00753   conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
00754   simParameters->receive_SimParameters(conv_msg);
00755 
00756   DebugM(4, "Getting Parameters\n");
00757   conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
00758   parameters->receive_Parameters(conv_msg);
00759 
00760   DebugM(4, "Getting Molecule\n");
00761   conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00762   // Modified by JLai -- 10.21.11
00763   molecule->receive_Molecule(conv_msg);
00764   if(simParameters->goForcesOn) {
00765     iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
00766     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00767     molecule->receive_GoMolecule(conv_msg);
00768   } 
00769   // End of modification
00770   DebugM(4, "Done Receiving\n");
00771 }
00772 
00773 void Node::namdOneSend() {
00774   node_simParameters = simParameters;
00775   node_parameters = parameters;
00776   node_molecule = molecule;
00777 
00778   MOStream *conv_msg;
00779   // I'm Pe(0) so I send what I know
00780   DebugM(4, "Sending SimParameters\n");  
00781   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
00782   simParameters->send_SimParameters(conv_msg);
00783 
00784   DebugM(4, "Sending Parameters\n");
00785   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
00786   parameters->send_Parameters(conv_msg);
00787 
00788   DebugM(4, "Sending Molecule\n");
00789   int bufSize = BUFSIZE;
00790   if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
00791   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
00792   // Modified by JLai -- 10.21.11
00793   molecule->send_Molecule(conv_msg);
00794   
00795   if(simParameters->goForcesOn) {
00796     iout << iINFO <<  "Master Node sending GoMolecule Information" << "\n" << endi;
00797     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
00798     molecule->send_GoMolecule(conv_msg);
00799   } // End of modification
00800 }
00801 
00802 // Initial thread setup
00803 
00804 void Node::threadInit() {
00805   // Thread initialization
00806   if (CthImplemented()) {
00807     CthSetStrategyDefault(CthSelf());
00808   } else {
00809     NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
00810   }
00811 }
00812 
00813 //
00814 void Node::buildSequencers() {
00815   HomePatchList *hpl = PatchMap::Object()->homePatchList();
00816   ResizeArrayIter<HomePatchElem> ai(*hpl);
00817 
00818   // Controller object is only on Pe(0)
00819   if ( ! CkMyPe() ) {
00820     Controller *controller = new Controller(state);
00821     state->useController(controller);
00822   }
00823 
00824   // Assign Sequencer to all HomePatch(es)
00825   for (ai=ai.begin(); ai != ai.end(); ai++) {
00826     HomePatch *patch = (*ai).patch;
00827     Sequencer *sequencer = new Sequencer(patch);
00828     patch->useSequencer(sequencer);
00829   }
00830 }
00831 
00832 
00833 
00834 //-----------------------------------------------------------------------
00835 // Node run() - broadcast to all nodes
00836 //-----------------------------------------------------------------------
00837 void Node::messageRun() {
00838   (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
00839 }
00840 
00841 
00842 //-----------------------------------------------------------------------
00843 // run(void) runs the specified simulation for the specified number of
00844 // steps, overriding the contents of the configuration file
00845 //-----------------------------------------------------------------------
00846 void Node::run()
00847 {
00848   // Start Controller (aka scalar Sequencer) on Pe(0)
00849 //  printf("\n\n I am in Node.C in run method about to call  state->runController\n\n");
00850   if ( ! CkMyPe() ) {
00851     state->runController();
00852   }
00853 
00854   DebugM(4, "Starting Sequencers\n");
00855   // Run Sequencer on each HomePatch - i.e. start simulation
00856   HomePatchList *hpl = PatchMap::Object()->homePatchList();
00857   ResizeArrayIter<HomePatchElem> ai(*hpl);
00858   for (ai=ai.begin(); ai != ai.end(); ai++) {
00859     HomePatch *patch = (*ai).patch;
00860 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
00861     patch->runSequencer();
00862   }
00863 
00864   if (!CkMyPe()) {
00865     double newTime = CmiWallTimer();
00866     iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00867          << newTime - startupTime << " s, "
00868          << memusage_MB() << " MB of memory in use\n";
00869     iout << iINFO << "Finished startup at " << newTime << " s, "
00870          << memusage_MB() << " MB of memory in use\n\n" << endi;
00871     fflush(stdout);
00872   }
00873   
00874 }
00875 
00876 
00877 //-----------------------------------------------------------------------
00878 // Node scriptBarrier() - twiddle parameters with simulation halted
00879 //-----------------------------------------------------------------------
00880 
00881 void Node::enableScriptBarrier() {
00882   CkStartQD(CkIndex_Node::scriptBarrier((CkQdMsg*)0),&thishandle);
00883 }
00884 
00885 void Node::scriptBarrier(CkQdMsg *qmsg) {
00886   delete qmsg;
00887   //script->awaken();
00888 }
00889 
00890 void Node::scriptParam(ScriptParamMsg *msg) {
00891   simParameters->scriptSet(msg->param,msg->value);
00892   delete msg;
00893 }
00894 
00895 void Node::reloadCharges(const char *filename) {
00896   FILE *file = fopen(filename,"r");
00897   if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
00898 
00899   int n = molecule->numAtoms;
00900   float *charge = new float[n];
00901 
00902   for ( int i = 0; i < n; ++i ) {
00903     if ( ! fscanf(file,"%f",&charge[i]) )
00904       NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
00905   }
00906 
00907   fclose(file);
00908   CProxy_Node(thisgroup).reloadCharges(charge,n);
00909   delete [] charge;
00910 }
00911 
00912 void Node::reloadCharges(float charge[], int n) {
00913   molecule->reloadCharges(charge,n);
00914 }
00915 
00916 
00917 // BEGIN gf
00918 void Node::reloadGridforceGrid(const char * key) {
00919     DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
00920     
00921     int gridnum;
00922     MGridforceParams *mgridParams;
00923     if (key == NULL) {
00924         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
00925         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
00926     } else {
00927         gridnum = simParameters->mgridforcelist.index_for_key(key);
00928         mgridParams = simParameters->mgridforcelist.find_key(key);
00929     }
00930     
00931     if (gridnum < 0 || mgridParams == NULL) {
00932         NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
00933     }
00934     
00935     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
00936     if (grid == NULL) {
00937         NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
00938     }
00939     grid->reinitialize(simParameters, mgridParams);
00940     
00941     CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
00942     
00943     DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
00944 }
00945 
00946 void Node::reloadGridforceGrid(int gridnum) {
00947     DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
00948     
00949     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
00950     if (grid == NULL) {
00951         NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
00952     }
00953     
00954     if (CkMyPe()) {
00955         // not node 0 -> receive grid
00956         if (CmiMyRank()) return;
00957         
00958         DebugM(4, "Receiving grid\n");
00959         
00960         delete grid;
00961         
00962         MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
00963         grid = GridforceGrid::unpack_grid(gridnum, msg);
00964         molecule->set_gridfrc_grid(gridnum, grid);
00965         delete msg;
00966     } else {
00967         // node 0 -> send grid
00968         DebugM(4, "Sending grid\n");
00969         
00970         MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
00971         GridforceGrid::pack_grid(grid, msg);
00972         msg->end();
00973         delete msg;
00974     }
00975     
00976     DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
00977 }
00978 // END gf
00979 
00980 
00981 void Node::sendEnableExitScheduler(void) {
00982   //CmiPrintf("sendEnableExitScheduler\n");
00983   CkQdMsg *msg = new CkQdMsg;
00984   CProxy_Node nodeProxy(thisgroup);
00985   nodeProxy[0].recvEnableExitScheduler(msg);
00986 }
00987 
00988 void Node::recvEnableExitScheduler(CkQdMsg *msg) {
00989   //CmiPrintf("recvEnableExitScheduler\n");
00990   delete msg;
00991   enableExitScheduler();
00992 }
00993 
00994 void Node::enableExitScheduler(void) {
00995   if ( CkMyPe() ) {
00996     sendEnableExitScheduler();
00997   } else {
00998     CkStartQD(CkIndex_Node::exitScheduler((CkQdMsg*)0),&thishandle);
00999   }
01000 }
01001 
01002 void Node::exitScheduler(CkQdMsg *msg) {
01003   //CmiPrintf("exitScheduler %d\n",CkMyPe());
01004   CsdExitScheduler();
01005   delete msg;
01006 }
01007 
01008 void Node::sendEnableEarlyExit(void) {
01009   CkQdMsg *msg = new CkQdMsg;
01010   CProxy_Node nodeProxy(thisgroup);
01011   nodeProxy[0].recvEnableEarlyExit(msg);
01012 }
01013 
01014 void Node::recvEnableEarlyExit(CkQdMsg *msg) {
01015   delete msg;
01016   enableEarlyExit();
01017 }
01018 
01019 void Node::enableEarlyExit(void) {
01020   if ( CkMyPe() ) {
01021     sendEnableEarlyExit();
01022   } else {
01023     CkStartQD(CkIndex_Node::earlyExit((CkQdMsg*)0),&thishandle);
01024   }
01025 }
01026 
01027 void Node::earlyExit(CkQdMsg *msg) {
01028   iout << iERROR << "Exiting prematurely; see error messages above.\n" << endi;
01029   BackEnd::exit();
01030   delete msg;
01031 }
01032 
01033 
01034 //------------------------------------------------------------------------
01035 // Some odd utilities
01036 //------------------------------------------------------------------------
01037 void Node::saveMolDataPointers(NamdState *state)
01038 {
01039   this->molecule = state->molecule;
01040   this->parameters = state->parameters;
01041   this->simParameters = state->simParameters;
01042   this->configList = state->configList;
01043   this->pdb = state->pdb;
01044   this->state = state;
01045 }
01046 
01047 // entry methods for BG/P HPM (performance counters) library
01048 void Node::startHPM() {
01049 #if USE_HPM
01050   HPM_Start("500 steps", localRankOnNode);
01051 #endif
01052 }
01053 
01054 void Node::stopHPM() {
01055 #if USE_HPM
01056   HPM_Stop("500 steps", localRankOnNode);
01057   HPM_Print(CkMyPe(), localRankOnNode);
01058 #endif
01059 }
01060 
01061 void Node::traceBarrier(int turnOnTrace, int step){
01062         curTimeStep = step;
01063         if(turnOnTrace) traceBegin();
01064         else traceEnd();
01065 
01066 #if CHARM_VERSION > 60400
01067     if(turnOnTrace) CmiTurnOnStats();
01068     else CmiTurnOffStats();
01069 #endif
01070 
01071         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());    
01072         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01073         CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
01074         contribute(0, NULL, CkReduction::sum_int, cb);
01075         
01076 }
01077 
01078 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
01079         CmiAssert(CmiMyPe()==0);
01080         delete msg;     
01081         state->controller->resumeAfterTraceBarrier(curTimeStep);
01082 }
01083 
01084 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
01085 #ifdef MEASURE_NAMD_WITH_PAPI
01086         curMFlopStep = step;
01087         double results[NUM_PAPI_EVENTS];
01088 
01089         if(turnOnMeasure){              
01090                 PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
01091         }else{
01092                 long long counters[NUM_PAPI_EVENTS];
01093                 PAPI_read_counters(counters, NUM_PAPI_EVENTS);
01094                 results[0] = (double)counters[0]/1e6;
01095                 results[1] = (double)counters[1]/1e6;
01096                 PAPI_stop_counters(counters, NUM_PAPI_EVENTS);  
01097         }
01098         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());
01099         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01100         CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
01101         contribute(sizeof(double)*NUM_PAPI_EVENTS, &results, CkReduction::sum_double, cb);      
01102 #endif
01103 }
01104 
01105 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
01106 #ifdef MEASURE_NAMD_WITH_PAPI
01107         if(simParameters->papiMeasureStartStep != curMFlopStep) {
01108                 double *results = (double *)msg->getData();
01109                 int bstep = simParameters->papiMeasureStartStep;
01110                 int estep = bstep + simParameters->numPapiMeasureSteps;
01111                 if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
01112                         double totalFPIns = results[0];
01113                         if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
01114                         CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n", 
01115                                          bstep, estep, totalFPIns/CkNumPes());
01116                 }else{
01117                         char nameBuf[PAPI_MAX_STR_LEN];
01118                         CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ", 
01119                                          bstep, estep);
01120                         for(int i=0; i<NUM_PAPI_EVENTS; i++) {
01121                                 PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
01122                                 CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
01123                         }
01124                         CkPrintf("per processor\n");
01125                 }               
01126         }
01127         delete msg;     
01128         state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
01129 #endif
01130 }
01131 
01132 extern char *gNAMDBinaryName;
01133 void Node::outputPatchComputeMaps(const char *filename, int tag){
01134         if(!simParameters->outputMaps && !simParameters->simulateInitialMapping) return;
01135 
01136         int numpes = CkNumPes();
01137         int nodesize = CkMyNodeSize();
01138         if(simParameters->simulateInitialMapping) {
01139                 numpes = simParameters->simulatedPEs;
01140                 nodesize = simParameters->simulatedNodeSize;
01141         }
01142 
01143         char fname[128];
01144         sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
01145 
01146         FILE *fp = fopen(fname, "w");
01147         if(fp == NULL) {
01148                 NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
01149                 return;
01150         }
01151         PatchMap *pMap = PatchMap::Object();
01152         ComputeMap *cMap = ComputeMap::Object();
01153         int numPatches = pMap->numPatches();
01154         int numComputes = cMap->numComputes();
01155         fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes, 
01156                         pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
01157         //output PatchMap info
01158         for(int i=0; i<numPatches; i++) {
01159         #ifdef MEM_OPT_VERSION
01160                 fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
01161         #else
01162                 fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
01163         #endif
01164         }
01165 
01166         //output ComputeMap info
01167         for(int i=0; i<numComputes; i++) {              
01168                 fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));             
01169         }
01170 }
01171 
01172 
01173 //======================================================================
01174 // Private functions
01175 
01176 
01177 #include "Node.def.h"
01178 

Generated on Fri May 25 04:07:16 2012 for NAMD by  doxygen 1.3.9.1