#!/bin/csh
#
#  26 Mar 03 - MWS - C-shell script to execute GAMESS
#
#  Each user needs an alias to point to this common script,
#  such as placing this line in their C shell .login file:
#       alias gms 'nice +15 /u1/mike/gamess/rungms'
#
#  The script is executed by any user by typing
#       gms JOB VERNO NNODES >& JOB.log &
#  JOB    is the name of the 'JOB.inp' file to be executed,
#  VERNO  is the number of the executable you chose at 'lked' time,
#  NNODES is the number of nodes to be used.
#
#  The installer needs to 
#    a) choose the target for execution from: 
#           compaq-shmem, cray-t3e, fuji-pp, ibm-sp, mpich, necsx, sgi-mpi
#           or for most cases, choose the target as "sockets".
#       Select "sockets" if your compile time target was any of these:
#           compaq-axp, cray-pvp, hpux32, hpux64, ibm32, ibm64,
#           linux-ia64, linux-pc, macosx, sgi32, sgi64, sun32, sun64
#       as all of these systems use TCP/IP sockets.
#    b) choose a directory SCR where large temporary files can reside.
#       this should be fastest possible disk access, and very spacious.
#       Translation: do not put these files on a network file server!
#    c) perhaps change the location of the PUNCH and IRCDATA files below,
#       these along with the output should probably go to a file server.
#    d) if you have no interest in running in parallel, hardwire NNODES to 1.
#    e) customize the execution section for your target below, each has
#       its own list of further requirements.
#
set TARGET=sockets
set SCR=.
#
set JOB=$1       # name of the input file xxx.inp, give only the xxx part
set VERNO=$2     # revision number of the executable created by 'lked' step
set NNODES=$3    # number of compute processes to be run

# provide defaults if last two arguments are not given to this script
if (null$VERNO == null) set VERNO=00
if (null$NNODES == null) set NNODES=1
#
#  ---- the top section of the script is input and other file assignments ----
#
echo ----- GAMESS execution script -----
set master=`hostname`
echo This job is running on host $master at `date`
echo "Available scratch disk space (Kbyte units) at beginning of the job is"
if ($TARGET != necsx) then
   df -k $SCR
else
   df -c $SCR
endif

#  Grab a copy of the input file.
#  In the case of EXAMnn jobs, this file might be in the "tests" subdirectory.
if ($JOB:r.inp == $JOB) set JOB=$JOB:r      # strip off possible .inp
if (-e $JOB.inp) then
   set echo
   cp  $JOB.inp  $SCR/$JOB.F05
   unset echo
else
   if (-e tests/$JOB.inp) then
      set echo
      cp  tests/$JOB.inp  $SCR/$JOB.F05
      unset echo
   else
      echo "Input file $JOB.inp does not exist.  Please fix and resubmit."
      exit 4
   endif
endif

#  file assignments.
#  all binary files should be put on a node's local disk ($SCR directory), 
#  both ASCII punch files might be written by NFS to user's permanent disk
set echo
setenv  EXTBAS /dev/null
setenv IRCDATA $SCR/$JOB.irc
setenv   INPUT $SCR/$JOB.F05
setenv   PUNCH $SCR/$JOB.dat
setenv  AOINTS $SCR/$JOB.F08
setenv  MOINTS $SCR/$JOB.F09
setenv DICTNRY $SCR/$JOB.F10
setenv DRTFILE $SCR/$JOB.F11
setenv CIVECTR $SCR/$JOB.F12
setenv CASINTS $SCR/$JOB.F13
setenv  CIINTS $SCR/$JOB.F14
setenv  WORK15 $SCR/$JOB.F15
setenv  WORK16 $SCR/$JOB.F16
setenv CSFSAVE $SCR/$JOB.F17
setenv FOCKDER $SCR/$JOB.F18
setenv OVLPDER $SCR/$JOB.F19
setenv  DASORT $SCR/$JOB.F20
setenv DFTINTS $SCR/$JOB.F21
setenv DFTGRID $SCR/$JOB.F22
setenv  JKFILE $SCR/$JOB.F23
setenv  ORDINT $SCR/$JOB.F24
setenv  EFPIND $SCR/$JOB.F25
setenv PCMDATA $SCR/$JOB.F26                                     
setenv PCMINTS $SCR/$JOB.F27
setenv  MLTPL  $SCR/$JOB.F28
setenv  MLTPLT $SCR/$JOB.F29
setenv  DAFL30 $SCR/$JOB.F30
setenv  SOINTX $SCR/$JOB.F31
setenv  SOINTY $SCR/$JOB.F32
setenv  SOINTZ $SCR/$JOB.F33
setenv  SORESC $SCR/$JOB.F34
setenv   SIMEN $SCR/$JOB.simen
setenv  SIMCOR $SCR/$JOB.simcor
setenv GCILIST $SCR/$JOB.F37
setenv CIMOHSS $SCR/$JOB.F38
setenv SOCCDAT $SCR/$JOB.F40
setenv  AABB41 $SCR/$JOB.F41
setenv  BBAA42 $SCR/$JOB.F42
setenv  BBBB43 $SCR/$JOB.F43
setenv  MCQD50 $SCR/$JOB.F50
setenv  MCQD51 $SCR/$JOB.F51
setenv  MCQD52 $SCR/$JOB.F52
setenv  MCQD53 $SCR/$JOB.F53
setenv  MCQD54 $SCR/$JOB.F54
setenv  MCQD55 $SCR/$JOB.F55
setenv  MCQD56 $SCR/$JOB.F56
setenv  MCQD57 $SCR/$JOB.F57
setenv  MCQD58 $SCR/$JOB.F58
setenv  MCQD59 $SCR/$JOB.F59
setenv  MCQD60 $SCR/$JOB.F60
setenv  MCQD61 $SCR/$JOB.F61
setenv  MCQD62 $SCR/$JOB.F62
setenv  MCQD63 $SCR/$JOB.F63
setenv  MCQD64 $SCR/$JOB.F64
setenv   GVVPT $SCR/$JOB.F69
setenv  CCREST $SCR/$JOB.F70
setenv  CCDIIS $SCR/$JOB.F71
setenv  CCINTS $SCR/$JOB.F72
setenv CCT1AMP $SCR/$JOB.F73
setenv CCT2AMP $SCR/$JOB.F74
setenv CCT3AMP $SCR/$JOB.F75
setenv    CCVM $SCR/$JOB.F76
setenv    CCVE $SCR/$JOB.F77
unset echo

if ((-e $PUNCH) || (-e $IRCDATA)) then
   echo Please rename/erase $PUNCH and/or $IRCDATA, and resubmit
   exit 4
endif

#  ---- the middle section of the script is to execute GAMESS ----
#
#   Most workstations use TCP/IP sockets, and therefore execute
#   according to the following clause.  The installer must
#      a) Set the path to point to the DDIKICK and GAMESS executables.
#      b) Build the HOSTLIST variable as a word separated string, i.e. ()'s.
#         There should be one host name for every compute process that is
#         to be run.  DDIKICK will automatically generate a set of data
#         server processes on the same hosts.
#   An extended explanation of the arguments to ddikick.x can be found
#   in the file PROG.DOC, if you have any trouble executing.
#
if ($TARGET == sockets) then
   set os=`uname`
   if ($os == AIX)    set GMSPATH=/u1/mike/gamess
   if ($os == Darwin) set GMSPATH=/Users/tcblaptop/markus/gamess
   if ($os == HP-UX)  set GMSPATH=/k/mike/gamess
   if ($os == IRIX)   set GMSPATH=/home/hbar1/people/schmidt/gamess
   if ($os == IRIX64) set GMSPATH=/home/hbar1/people/schmidt/gamess
   if ($os == Linux)  set GMSPATH=./
   if ($os == OSF1)   set GMSPATH=/au/mike/gamess
   if ($os == SunOS)  set GMSPATH=/hf/mike/gamess
#       special compilation with Power3 instruction set for our newest IBMs
   if (`hostname` == pb.fi.ameslab.gov) set GMSPATH=/pb/mike/gamess
   if (`hostname` == bi.fi.ameslab.gov) set GMSPATH=/pb/mike/gamess
   if (`hostname` == po.fi.ameslab.gov) set GMSPATH=/pb/mike/gamess
#       special compilation for my old Sun
   if (`hostname` == sc.fi.ameslab.gov) set GMSPATH=/sc/mike/gamess

#       Five examples of how to build the HOSTLIST are shown....
#
#       1. Sequential execution is sure to be on this very same host
   if ($NNODES == 1) set HOSTLIST=(`hostname`)
   #       We'll default 2 processor runs to the same host. Change this
   #       if running over a network or if you only have a single CPU machine.
      if ($NNODES == 2) set HOSTLIST=(localhost localhost)
#
#       2. This is an example of how to run on a 8-way SMP enclosure
   if ((`hostname` == ti.fi.ameslab.gov) && ($NNODES > 1)) then
      set HOSTLIST=()
      set nmax=$NNODES
      if ($nmax > 8) set nmax=8
      @ NODE=1
      while ($NODE <= $nmax)
         set HOSTLIST=($HOSTLIST `hostname`)
         @ NODE++
      end
      unset $NODE
   endif
#
#       3. We use 16 Intel Pentium 2 machines in a Beowulf-type cluster.
#          Their host names fall into the pattern fly1 to fly16.
   if (`hostname` == fly1.fi.ameslab.gov) then
      set HOSTLIST=()
      set nmax=$NNODES
      if ($nmax > 16) set nmax=16
      @ NODE=1
      while ($NODE <= $nmax)
         set HOSTLIST=($HOSTLIST fly$NODE)
         @ NODE++
      end
      unset $NODE
   endif
#
#       4. We use six Pentium2 systems as a 6-ways parallel queue.
#          Since their names never change, we just can just specify them.
   if (`hostname` == ga.fi.ameslab.gov) then
      set HOSTLIST=(ga ge as se br sb)
   endif
#
#       5. Scalable Computing Lab's clusters running PBS batch queues.
#          Here the task is to manipulate the dynamically assigned host
#          names into the HOSTLIST string for the kickoff program.
#          The two clusters run different versions of PBS, and therefore
#          return line-by-line host lists by two different file names.
#          Furthermore, the IBM cluster gives already the names of the
#          Gigabit Ethernet adapters, so we just need to parse the file.
#          However, the Alpha cluster gives its Fast Ethernet names like
#          "alpha15" but we want its Myrinet, which is "alpha15.myri".
   if ($?PBS_JOBID) then
   set os=`uname`
   if ($os == AIX)  set NODEFILE=$SCR/hostlist
   if ($os == AIX)  set suffix=''
   if ($os == OSF1) set NODEFILE=$PBS_NODEFILE
   if ($os == OSF1) set suffix='.myri'
   set GMSPATH=/usr/local/bin/gamess
#
   if ($NNODES > 1) then
      set HOSTLIST=()
      set nmax=`wc -l $NODEFILE`
      set nmax=$nmax[1]
      if ($NNODES < $nmax) set nmax=$NNODES
      @ NODE=1
      while ($NODE <= $nmax)
         set HOSTLIST=($HOSTLIST `sed -n -e "$NODE p" $PBS_NODEFILE`$suffix)
         @ NODE++
      end
   endif

   unset $NODE
   unset $NODEFILE
   unset $suffix
   endif
#
#      With the HOSTLIST variable now in hand, we can use the kickoff
#      program to initiate the GAMESS process(es) on all nodes.
#
   if ((-x $GMSPATH/gamess.$VERNO.x) && (-x $GMSPATH/ddikick.x)) then
      set path=($GMSPATH $path)
      set echo
      ddikick.x $JOB $GMSPATH gamess.$VERNO.x $SCR $NNODES $HOSTLIST < /dev/null
      unset echo
   else
      echo The GAMESS executable gamess.$VERNO.x or else
      echo the DDIKICK executable ddikick.x
      echo could not be found in directory $GMSPATH,
      echo or else they did not properly link to executable permission.
      exit 8
   endif
endif

#   Compaq running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == compaq-shmem) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   prun -n $NNODES $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif
 
#   Cray T3E wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == cray-t3e) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   mpprun -n $NNODES $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif

#   Fujitsu PrimePower want you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == fuji-pp) then
   set GMSPATH=/some/full/path/gamess
   set echo
   mpiexec -n $NNODES -mode limited $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif

#   The IBM SP wants you to
#      a) set the path to point to the GAMESS executable
#      b) use the same work area name on all nodes (SCR directory).
#         preferably this is a disk local to each node, not a piofs.
#      c) define hosts in a host file, which might be hardwired as here,
#         or generated from a variable such as LOADL_PROCESSOR_LIST
#         that is provided by the Load Leveler system.
#   Note: two processes must be run per node, compute processes must
#   be generated on the nodes p1, p2, p3, ..., pn, and then back to p1, 
#   p2, ..., pn for the data server processes.  Both processes are the
#   same executable, gamess.NN.x, the first one will do the computing
#   of quantum chemistry, and the second one will take up the task of 
#   distributed data management, performing almost no computing at all.
#
#   Please note that we have an official LoadLeveler script being
#   used on the SP at the Maui computer center.
#   If you want to use LL, ask for this script, mike@si.fi.ameslab.gov
#
if ($TARGET == ibm-sp) then
   set path=($path /u1/mike/gamess)
   setenv NLSPATH /usr/lib/nls/msg/%L/%N:/usr/lib/nls/msg/prime/%N
   setenv LANG En_US
#
#     the following is how SP accomplishes "chdir $SCR" on each node.
   setenv MP_REMOTEDIR ~/scr/$JOB.remotedir
   echo "echo $SCR" > $MP_REMOTEDIR
   chmod 755 $MP_REMOTEDIR
#
#      Now we build a list of host names on which we are to run.
#      GAMESS needs to double up the number of processes, as the
#      first process group computes, the second are data servers.
#      Therefore we need to get a file containing a list of hosts
#      followed by a second copy of the host list.
#
   setenv HOSTFILE $SCR/$JOB.hosts

#      host names are probably selected by LoadLeveler, dynamically.
#      These names are supplied in an environment variable.
#
   set LL_hosts=($LOADL_PROCESSOR_LIST)
   if (-e $HOSTFILE) rm -f $HOSTFILE
   @ i=0
   while ($i < $#LL_hosts)
      @ i++
      echo $LL_hosts[$i] >> $HOSTFILE
   end
   @ i=0
   while ($i < $#LL_hosts)
      @ i++
      echo $LL_hosts[$i] >> $HOSTFILE
   end
   unset LL_hosts

   @ NPROCS = $NNODES + $NNODES
   echo Running $NNODES compute processes and $NNODES data server processes...
   set echo


#     if the command 'lslpp ppe.poe' replies with version 2.4 installed,
#     uncomment the following and select 'user space' communications.
#--setenv MP_CPU_USE        unique       # new in POE 2.4
#--setenv MP_ADAPTER_US     shared       # new in POE 2.4
#--setenv MP_MSG_API        mpi          # new in POE 2.4
#--setenv MP_TASKS_PER_NODE 2            # new in POE 2.4
#--setenv MP_NODES          $NNODES      # new in POE 2.4

   setenv MP_PROCS          $NPROCS
   setenv MP_HOSTFILE       $HOSTFILE
   setenv MP_EUILIB         ip           # should be 'us' if POE 2.4 installed
   setenv MP_EUIDEVICE      css0
   setenv MP_PGMMODEL       mpmd

   poe gamess.$VERNO.x -stdinmode none
   unset echo
endif

#   This is for a generic MPICH version
#      a) set the path to point to the GAMESS executable and 'mpirun'
#      b) define a HOSTFILE properly.
#         We use 16 Intel Pentium 2 machines in a Beowulf-type cluster,
#         with host names in the pattern fly1 to fly16.
#         We need to prepare a doubled up host file list, naming all
#         the processors to be used for compute processes, and then
#         repeating it (in the identical order) for the data servers.
#   If you are running on a workstation cluster, you should be using
#   our TCP/IP socket interface, as MPICH will undoubtedly be slower.
#
if ($TARGET == mpich) then
   if (`hostname` == fly1.fi.ameslab.gov) then
      set HOSTFILE=$SCR/$JOB.hosts
      if (-e $HOSTFILE) rm $HOSTFILE
      touch $HOSTFILE
      set nmax=$NNODES
      if ($nmax > 16) set nmax=16
      @ NODE=1
      while ($NODE <= $nmax)
         echo fly$NODE.fi.ameslab.gov >> $HOSTFILE
         @ NODE++
      end
      @ NODE=1
      while ($NODE <= $nmax)
         echo fly$NODE.fi.ameslab.gov >> $HOSTFILE
         @ NODE++
      end
   endif
   @ NPROCS = $NNODES + $NNODES

   set path=(/spider/root/mpich/bin /spider/mike/gamess $path)

   set echo
   mpirun -np $NPROCS -machinefile $HOSTFILE gamess.$VERNO.x
   unset echo
endif

#   NEC SX Series wants you to
#      a) set the path variable to point to the GAMESS executable
#   Note that this version is running in serial mode.
if ($TARGET == necsx) then
   set GMSPATH=/u1/mike/gamess

   chdir $SCR
   setenv F_RECLUNIT BYTE
   setenv F_ABORT YES
   setenv F_ERROPT1 252,252,2,2,1,1,2,1
   setenv F_PROGINF detail
   setenv F_SETBUF 4096
   set echo
   $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif

#   MPI on SGI wants you to
#      a) set the path variable to point to the GAMESS executable
#   This did not work using 64 bit integers, but was OK for sgi32.
if ($TARGET == sgi-mpi) then
   set GMSPATH=/home/hbar1/people/schmidt/gamess
   echo Running $NNODES compute processes and $NNODES data server processess...
   @ NPROCS = $NNODES + $NNODES
   chdir $SCR
   set echo
   mpirun -np $NPROCS $GMSPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif
#
#   Windows
#      a) set the path variable to point to the GAMESS executable
if ($TARGET == win32) then
   set GMSPATH=object
   set echo
   $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif
#
#  ---- the bottom section of the script is to clean up all disk files ----
#
echo ----- accounting info -----
date
#
#   Clean up the master's scratch directory.
#
echo Files used on the master node $master were:
ls -lF $SCR/$JOB.*
rm -f  $SCR/$JOB.F*
#
#   Clean up scratch directory of remote nodes.
#   This may not be necessary, e.g. on a T3E where all files are in the
#   same directory, and just got cleaned out by the previous 'rm'.
#   Some batch queue managers may also provide cleaned out scratch
#   directories.
#   If you run on some system other than 'sockets', you may need to 
#   copy the loop shown below for use on your systems, possibly using
#   something like 'ssh' instead of the 'rsh' commands to clean your
#   other node's disks out.  
#   The 'lasthost' business prevents multiple cleanup tries on SMP boxes.
#
#if ($TARGET == sockets) then
#   set nmax=${#HOSTLIST}
#   set lasthost=$master
#   @ n=2   # master already cleaned above
#   while ($n <= $nmax)
#      set host=$HOSTLIST[$n]
#      if ($host != $lasthost) then
#         echo Files from $host are:
#         rsh $host -l $USER -n "ls -l $SCR/$JOB.*"
#         rsh $host -l $USER -n "rm -f $SCR/$JOB.F*"
#      endif
#      set lasthost=$host
#      @ n++
#   end
#endif
#
#   These two can run through the first half of the host file.
#
if (($TARGET == ibm-sp) || ($TARGET == mpich)) then
   set nmax=$NNODES
   set lasthost=$master
   @ n=2   # we already cleaned up the master node just above.
   while ($n <= $nmax)
      set host=`sed -n -e "$n p" $HOSTFILE`
      if ($host != $lasthost) then
         echo Files used on node $host were:
         rsh $host -l $USER -n "ls -l $SCR/$JOB.*"
         rsh $host -l $USER -n "rm -f $SCR/$JOB.F*"
      endif
      @ n++
   end
   rm -f $HOSTFILE
   if ($TARGET == ibm-sp) rm -f $MP_REMOTEDIR
endif

#  and this is the end
time
exit