ComputeMoa.C

Go to the documentation of this file.
00001 /* *
00002  * ***  Copyright (c) 1995, 1996, 1997, 1998, 1999, 2000 by
00003  * ***  The Board of Trustees of the University of Illinois.
00004  * ***  All rights reserved.
00005  * **
00006  * =====================================================================================
00007  *
00008  *       Filename:  ComputeMoa.C
00009  *
00010  *    Description:  
00011  *
00012  *        Version:  Stub File
00013  *        Created:  03/08/2012 03:55:27 PM
00014  *       Revision:  
00015  *       Compiler:  charm++ 
00016  *
00017  *         Author:  Christopher B Harrison, Ph.D.
00018  *         Email:   charris5@gmail.com, char1@illinois.edu, char@ks.uiuc.edu
00019  *        Company:  
00020  *
00021  * =====================================================================================
00022  */
00023 
00024 #include "InfoStream.h"
00025 #include "Node.h"
00026 #include "PDB.h"
00027 #include "PatchMap.h"
00028 #include "PatchMap.inl"
00029 #include "ComputeMoa.h"
00030 #include "ComputeMoaMgr.decl.h"
00031 #include "PatchMgr.h"
00032 #include "Molecule.h"
00033 #include "ReductionMgr.h"
00034 #include "ComputeMgr.h"
00035 #include "ComputeMgr.decl.h"
00036 // #define DEBUGM
00037 #define MIN_DEBUG_LEVEL 3
00038 #include "Debug.h"
00039 #include "SimParameters.h"
00040 #include "WorkDistrib.h"
00041 #include "varsizemsg.h"
00042 #include <stdlib.h>
00043 #include <stdio.h>
00044 #include <errno.h>
00045 #include "pup_stl.h"
00046 
00047 #ifdef CHARM_HAS_MSA
00048 
00049 #ifdef OPENATOM_VERSION
00050 
00051 
00052 void MoaData::pup(PUP::er &p)
00053 {
00054   p|K1, p|K2, p|K3;
00055   p|order;
00056   p|orig_x, p|orig_y, p|orig_z;
00057 
00058   p|num_clients;
00059   p|num_clients_qh, p|num_clients_bh;
00060 
00061   p|qh; 
00062   p|sh;
00063   p|bh;
00064 
00065   p|k1r, p|k2r, p|k3r;
00066   p|gbqs;
00067   p|hasLQ;
00068   p|hasLB; 
00069   p|hasLS;
00070 }
00071 
00072 void MoaData::print()
00073 {
00074 #if 0
00075   printf("MoaData:\n");
00076   printf("K1,K2,K3 = %d, %d, %d\n", K1, K2, K3);
00077   printf("order = %d\n", order);
00078   printf("origin = %g %g %g\n", orig_x, orig_y, orig_z);
00079 #endif
00080 }
00081 
00082 class ComputeMoaMgr : public CBase_ComputeMoaMgr {
00083 public:
00084   ComputeMoaMgr();                    // entry
00085   ~ComputeMoaMgr();
00086 
00087   CkCallback doneMoa;
00088 
00089   void initialize(CkQdMsg *);         // entry with message
00090 
00091   void setCompute(ComputeMoa *c) { moaCompute = c; }  // local
00092 
00093   void recvMoaData(const MoaData &);  // entry with parameter marshalling
00094 
00095   void initWorkers(CkQdMsg *);        // entry with message
00096   void startWorkers(CkQdMsg *);       // entry with message
00097   
00098   void recvQ(int g, int numgrids, int nq, float qgrid[nq], CkCallback doneMoa);
00099   
00100   void recvB(int K2_start, int K2_end, int K3_start, int K3_end, int K1_len, int K2_len, int K3_len, double bm1[K1_len], double bm2[K2_len], double bm3[K3_len], int order);     // entry with message
00101   
00102   const MoaData &getMoaData() const { return moaData; }  // local
00103 
00104 private:
00105   CProxy_ComputeMoaMgr moaProxy;
00106   ComputeMoa *moaCompute;
00107 
00108   MoaData moaData;
00109 
00110   CProxy_MoaPatch moaPatchProxy;   // 1D chare array 
00111   CProxy_MoaS moaSProxy;   // 1D chare array 
00112   
00113   Moa3Grid::Write qhwrit // write charge grid
00114   Moa3Grid::Accum qhacc;  // accumulate charge grid 
00115   Moa3Grid::Read qhread;  // read charge grid 
00116   Moa3Grid::Write shwrit;  // writes S grid 
00117   Moa3Grid::Accum shacc;  // accumulate S grid 
00118   Moa3Grid::Read shread;  // read S grid 
00119   Moa3Grid::Write bhwrit; // writes b1, b2, b3 values
00120   Moa1Grid::Read bhread; // read b1, b2, b3 values
00121 };
00122 
00123 // class MoaPatch : public CBase_MoaPatch {
00124 // public:
00125 //   MoaPatch(Moa3Grid &qh, Moa3Grid &eh);
00126 //   MoaPatch(CkMigrateMessage *m) { }
00127 //   void compute();
00128 // private:
00129 // };
00130 
00131 class MoaS : public CBase_MoaS {
00132 public:
00133   MoaS(Moa3Grid &q, Moa3Grid &b);
00134   MoaS(CkMigrateMessage *m) { }
00135   ~MoaS();
00136   void compute();
00137 private:
00138   Moa3Grid qh, bh;
00139   Moa3Grid::Write shwrite;
00140   Moa3Grid::Read qhread, bhread;
00141   int moaS_setup;
00142   int lia, lja, lka; // local indices of sub-lattice
00143   int lib, ljb, lkb; // local indices of sub-lattice
00144   SubmitReduction *reduction;
00145 };
00146 
00147 
00148 ComputeMoa::ComputeMoa(ComputeID c) :
00149   ComputeHomePatches(c)
00150 {
00151 #ifdef OPENATOM_VERSION_DEBUG
00152   CkPrintf("Instantiated ComputeMoa %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00153 #endif //OPENATOM_VERSION_DEBUG
00154 
00155   CProxy_ComputeMoaMgr::ckLocalBranch(
00156       CkpvAccess(BOCclass_group).computeMoaMgr)->setCompute(this);
00157   SimParameters *simParams = Node::Object()->simParameters;
00158   // reduction = ReductionMgr::Object()->willSubmit(REDUCTIONS_BASIC);
00159 }
00160 
00161 ComputeMoa::~ComputeMoa()
00162 {
00163 }
00164 
00165 void ComputeMoa::doWork()
00166 {
00167 #ifdef OPENATOM_VERSION_DEBUG
00168   CkPrintf("Starting ComputeMoa::doWork() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00169 #endif //OPENATOM_VERSION_DEBUG
00170 
00171   // allocate message
00172 
00173   CProxy_ComputeMoaMgr moaProxy(CkpvAccess(BOCclass_group).computeMoaMgr);
00174   moaS_Proxy[CkMyPe()].compute(new CkQdMsg);
00175 
00176 }
00177 
00178 ComputeMoaMgr::ComputeMoaMgr() :
00179   moaProxy(thisgroup), moaCompute(0)
00180 {
00181 #ifdef OPENATOM_VERSION_DEBUG
00182   CkPrintf("Instantiating ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00183 #endif //OPENATOM_VERSION_DEBUG
00184   CkpvAccess(BOCclass_group).computeMoaMgr = thisgroup;
00185 }
00186 
00187 ComputeMoaMgr::~ComputeMoaMgr()
00188 {
00189 #ifdef OPENATOM_VERSION_DEBUG
00190   CkPrintf("Destructing ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00191 #endif //OPENATOM_VERSION_DEBUG
00192 
00193 }
00194 
00195 
00196 void ComputeMoaMgr::initialize(CkQdMsg *msg)
00197 {
00198 #ifdef OPENATOM_VERSION_DEBUG
00199   CkPrintf("Initializing ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00200 #endif //OPENATOM_VERSION_DEBUG
00201 
00202   delete msg;
00203   if (CkMyPe() != 0) return;  // initialize only on PE 0, broadcast MoaData
00204 
00205   // initialize MOA here
00206   SimParameters *simParams = Node::Object()->simParameters;
00207   Molecule *mol = Node::Object()->molecule;
00208 
00209   // setup grids and vectors
00210   MoaData &p = moaData;
00211   // p.hasLQ=0;
00212   // p.hasLB=0;
00213 
00214   extern int isPmeProcessor(int);
00215   int i, num_clients=0;
00216   for (i=0;i<CkNumPes(), i++)
00217   {
00218     if (isPmeProcessor(i)) p.num_clients++;
00219   }
00220 
00221   // RESTORE VDW PARAMETERS 
00222 
00223   // RESTORE MOLECULE OBJECT PARAMETERS fepAtomFlag atoms vdw 0 and charge bCol 
00224 
00225   int done, alldone;
00226 
00227 
00228 #ifdef MOA_DEBUG
00229   CkPrintf("global num clients = %d\n", p.num_clients);
00230   for (i = 0;  i < p.num_clients;  i++) {
00231     CkPrintf("num clients bh[%d] = %d\n", i, p.num_clients_bh[i]);
00232     CkPrintf("num clients qh[%d] = %d\n", i, p.num_clients_qh[i]);
00233     CkPrintf("num clients sh[%d] = %d\n", i, p.num_clients_bh[i]);
00234   }
00235 #endif
00236 
00237   p.hasLB.resize(p.num_clients);
00238   p.hasLQ.resize(p.num_clients);
00239   p.bh.resize(p.num_clients);
00240   p.qh.resize(p.num_clients);
00241   p.sh.resize(p.num_clients);
00242 
00243   for (i = 0;  i < p.num_clients;  i++) {
00244     ia = p.k1r[i].nx;
00245     ib = p.k1r[i].ny; 
00246     ja = p.k2r[i].nx;
00247     jb = p.k2r[i].ny;
00248     ka = p.k3r[i].nx;
00249     kb = p.k3r[i].ny;
00250 
00251     p.bh[i] = Moa3Grid(ia, ib, ja, jb, ka, kb, p.num_clients_qh[i]);
00252     p.qh[i] = Moa3Grid(ia, ib, ja, jb, ka, kb, p.num_clients_qh[i]);
00253     p.sh[i] = Moa3Grid(ia, ib, ja, jb, ka, kb, p.num_clients_sh[i]);
00254 
00255   }
00256   moaProxy.recvMoaData(moaData);  // broadcast MoaData to chare group
00257 
00258 #ifdef OPENATOM_VERSION_DEBUG
00259   CkPrintf("Initialized ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00260 #endif //OPENATOM_VERSION_DEBUG
00261 
00262 }
00263 
00264 void ComputeMoaMgr::recvB(int K2_start, int K2_end, int K3_start, int K3_end, int K1_len, int K2_len, int K3_len, double bm1[K1_len], double bm2[K2_len], double bm3[K3_len], int order)
00265 {
00266 #ifdef OPENATOM_VERSION_DEBUG
00267   CkPrintf("Started recvB() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00268 #endif //OPENATOM_VERSION_DEBUG
00269 
00270 //  const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
00271 
00272   MoaData &p = moaData;
00273   int j,m,n;
00274 
00275   p.k1r[thisIndex].nx = 0;
00276   p.k1r[thisIndex].ny = K1_len;
00277   p.k2r[thisIndex].nx = K2_start;
00278   p.k2r[thisIndex].ny = K2_end;
00279   p.k3r[thisIndex].nx = K3_start;
00280   p.k3r[thisIndex].ny = k3_end;
00281 
00282   p.bh[thisIndex].enroll();
00283   bhwrit = p.bh[thisIndex].syncToWrite();
00284   
00285   for ( j=0; j < K1_len; j++ ) {
00286     bhwrit.set(j,K2_start,K3_start) = bm1[j];
00287     for ( m=K2_start; m < K2_end; m++ ) {
00288       bhwrit.set(j,m,K3_start) = bm2[m];
00289       for ( n=K3_start; n < K3_end; n++ ) {
00290         bhwrit.set(j,m,n) = bm3[n];
00291       }
00292     }
00293   }
00294 
00295   bhread = bhwrit.syncToRead();
00296   
00297   p.hasLB[thisIndex]=1;
00298   p.gbqs.nx++;
00299   
00300 }
00301 
00302 void ComputeMoaMgr::recvQ(int g, int numgrids, int nq, float qgrid[nq], CkCallback doneMoa)
00303 {
00304 #ifdef OPENATOM_VERSION_DEBUG
00305   CkPrintf("Started recvQ() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00306 #endif //OPENATOM_VERSION_DEBUG
00307 
00308 //  const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
00309   MoaData &p = moaData;
00310 
00311   p.qh[g].enroll(p.num_clients_qh[thisIndex]);
00312   Eqhacc = p.qh[g].getInitialAccum();
00313 
00314   int i,j,m,n;
00315   
00316   int K1_len = p.k1r[thisIndex].ny;
00317   int K2s = p.k2r[thisIndex].nx;
00318   int K2e = p.k2r[thisIndex].ny;
00319   int K3s = p.k3r[thisIndex].nx;
00320   int K3e = p.k3r[thisIndex].ny;
00321   
00322   // qhwrit = p.qh[thisIndex].syncToWrite();
00323   bhread = p.bh[thisIndex].syncToRead();
00324 
00325 //  for ( i=0, i < nq, i+2) {
00326 //    qhrwrit.set(i)=qgrid[i];
00327 //    qhiwrit.set(i)=qgrid[i+1];
00328 //  }
00329 
00330   
00331   for ( j=CkIndex(); j < K1_len; j++ ) {
00332     qhwrit.set(j,K2s,K3s) = qgrid[j];
00333     for ( m=K2s; m < K2e; m++ ) {
00334       bhwrit.set(j,m,K3s) = qgrid[m];
00335       for ( n=K3s; n < K3e; n++ ) {
00336         bhwrit.set(j,m,n) = qgrid[n];
00337       }
00338     }
00339   }
00340 
00341  
00342   // for ( j = nq*g; j <= (nq * g + nq); j++) 
00343   for ( i = 0; i <= nq ; i++) {
00344     p.qhwrite[g][i] = qgrid[i];
00345   }
00346 
00347   if (g = numgrids) {
00348     p.gbqs.ny++; 
00349     p.hasLQ[thisIndex]=1;  
00350   }
00351 
00352 }
00353 
00354 void ComputeMoaMgr::sendS2OA()
00355 {
00356   // syncToRead BGrid, pack it
00357   // syncToRead SGrid, pack it
00358   // send msg to openatom containing BGrid and SGrid
00359   if (CkMyPe() != 0) return;
00360 
00361 #ifdef OPENATOM_VERSION_DEBUG
00362   CkPrintf("Started sendS2OA() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00363 #endif //OPENATOM_VERSION_DEBUG
00364 
00365   CProxy_CP_LargeSP_RhoGSpacePlane oaRGSP_Proxy;
00366 
00367   CkCallback resumeMoa(CkIndex_ComputeMoaMgr::recvSGrid(), thishandle);
00368   oaRGSP_Proxy[CkMyPe()].recvMDSg(g, ns, sg, resumeMoa);
00369 
00370 }
00371 
00372 void ComputeMoaMgr::recvSGrid(int g, int nq, double oaSg[nq])
00373 {
00374   if (CkMyPe() != 0) return;
00375 #ifdef OPENATOM_VERSION_DEBUG
00376   CkPrintf("Started recvSGrid() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00377 #endif //OPENATOM_VERSION_DEBUG
00378 
00379   MoaData &p = moaData;
00380   numgrids = p.numgrids;
00381 
00382 
00383 }
00384 
00385 void ComputeMoaMgr::initWorkers(CkQdMsg *msg)
00386 {
00387   delete msg;
00388   //printf("MOA initWorkers PE=%d\n", CkMyPe());
00389   if (CkMyPe() != 0) return;
00390   // PE 0 creates the compute chare arrays
00391 
00392   MoaData &p = moaData;
00393   int n;
00394   moaPatchProxy = CProxy_MoaPatch::ckNew();  // create empty chare array
00395 #ifdef OPENATOM_VERSION_DEBUG
00396   CkPrintf("Instantiated MoaPatch chare %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00397 #endif // OPENATOM_VERSION_DEBUG
00398 
00399   moaSProxy = CProxy_MoaS::ckNew(p.qh[0], p.eh[0], p.num_energy_chares);
00400 
00401 #ifdef OPENATOM_VERSION_DEBUG
00402   CkPrintf("Created MoaS chare %d at PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00403 #endif // OPENATOM_VERSION_DEBUG
00404 }
00405 
00406 
00407 void ComputeMoaMgr::startWorkers(CkQdMsg *msg)
00408 {
00409   delete msg;
00410   if (CkMyPe() != 0) return;
00411   // only PE 0 can start workers;
00412 #ifdef OPENATOM_VERSION_DEBUG
00413   CkPrintf("Starting MoaMgr Workers on chare %d at PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00414 #endif // OPENATOM_VERSION_DEBUG
00415 
00416   // moaPatchProxy.compute();
00417   moaSProxy.compute();
00418 }
00419 
00420 
00421 //  /* Archaic soln */
00422 //  MoaPatch::MoaPatch(Moa3Grid &qh, Moa3Grid &sh, Moa3Grid &bh)
00423 //  {
00424 //  #ifdef OPENATOM_VERSION_DEBUG
00425 //    CkPrintf("Instantiated MoaPatch %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00426 //  #endif // OPENATOM_VERSION_DEBUG
00427 //  
00428 //    const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch(CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
00429 //    int pIndex = thisIndex;
00430 //    const Int3 &dg = p.dim_gridcutoff_chares[pIndex];
00431 //    calcS = CProxy_MoaSGrid::ckNew(pIndex, qh, eh, dg.nx, dg.ny, dg.nz);
00432 //  }
00433 
00434 
00435 MoaS::MoaS(Moa3Grid &qh, Moa3Grid &bh)
00436 {
00437 #ifdef OPENATOM_VERSION_DEBUG
00438   CkPrintf("Instantiating MoaS %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00439 #endif //OPENATOM_VERSION_DEBUG
00440   
00441   const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData(); 
00442 
00443   
00444   lia = p.k1r.nx;
00445   lib = p.k1r.ny;
00446   lja = p.k2r.nx;
00447   ljb = p.k2r.ny;
00448   lka = p.k3r.nx;
00449   lkb = p.k3r.ny;
00450   buff_len = (lib - lia) + (ljb - lja) + (lkb - lka);
00451   qhbuff = new float[buff_len];
00452   bhbuff = new int[buff_len];
00453 
00454   moaS_setup = 0;
00455 
00456   reduction = ReductionMgr:Object()->willSubmit(REDUCTIONS_BASIC);
00457 }
00458 
00459 MoaS::~MoaS()
00460 {
00461 #ifdef OPENATOM_VERSION_DEBUG
00462   CkPrintf("Destructing MoaS %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00463 #endif //OPENATOM_VERSION_DEBUG
00464 
00465 }
00466 
00467 MoaS::compute()
00468 {
00469 #ifdef OPENATOM_VERSION_DEBUG
00470   CkPrintf("Starting MoaS compute %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
00471 #endif //OPENATOM_VERSION_DEBUG
00472 
00473   const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData(); 
00474    
00475   if ( ! moaS_setup ) {
00476     bh.enroll(p.num_clients);
00477     qh[0].enroll(p.num_clients);
00478     sh.enroll(p.num_clients);
00479    
00480     bhread = bh.syncToRead();
00481     qhacc = qh.getInitialAccum();
00482     qhread = qhacc.syncToRead();
00483     shwrit = sh.getInitialWrite();
00484     moaS_setup = 1;
00485   }
00486 
00487   bhread = bh.syncToRead();
00488 
00489   int j,m,n,index;
00490 
00491   for (index = 0, j = lia; j <= lib; j++) {
00492     for (m = lja; m <= ljb; m++) {
00493       for (n = lka; n <= lkb; n++, index++) {
00494         bhbuff[index] = bhread.get(j,m,n);
00495       }
00496     }
00497   }
00498   
00499   qhacc = qhread.syncToEAccum();
00500   qhread = qhacc.syncToRead();
00501   for (index = 0, j = lia; j <= lib; j++) {
00502     for (m = lja; m <= ljb; m++) {
00503       for (n = lka; n <= lkb; n++, index++) {
00504         qhbuff[index] = qhread.get(j,m,n);
00505       }
00506     }
00507   }
00508 
00509   shwrit = sh.getInitialWrite();
00510   for (index = 0, j = lia; j <= lib; j++) {
00511     for (m = lja; m <= ljb; m++) {
00512       for (n = lka; n <= lkb; n++, index++) {
00513         shwrite.set(j,m,n) = qhbuff[index] * bhbuff[index] * bhbuff[index+1] * bhbuff[index+2]
00514       }
00515     }
00516   }
00517 
00518 }
00519 
00520 
00521 #include "ComputeMoaMgr.def.h"
00522 
00523 #endif // OPENTATOM_VERSION
00524 
00525 #endif // CHARM_HAS_MSA
00526 
00527 

Generated on Tue Sep 26 01:17:11 2017 for NAMD by  doxygen 1.4.7