#!/bin/tcsh

set PPN = 32

if ( $# < 3 ) then
  echo "This script requires three arguments:"
  echo "  NAMD input file"
  echo "  NAMD log file"
  echo "  number of nodes (number of cores / $PPN)"
  echo "  processes per node (defaults to 4, choices are 1,2,3,4,31,32)"
  echo '  queue (defaults to "normal", other option is "test")'
  echo "  replica or other args (optional)"
  exit -1
endif

set CONFFILE = $1
shift
set LOGFILE = $1
shift
set NUMNODES = $1
shift

set REPLICAS = 1
set REPARGS = ""
set PROCSPERNODE = 4
set ARCH = "-ugni-smp-BlueWaters"

if ( $# && $1 !~ *[a-z]* ) then
    set PROCSPERNODE = $1
  shift
endif

set QUEUE = normal
set TIME = 24:00:00

if ( $# ) then
  if ( "$1" == "test" ) then
    set QUEUE = debug
    set TIME = 00:30:00
    if ( $NUMNODES > 7000 ) set QUEUE = high
    shift
  endif
endif

while ( $# )
  if ( "$1" == "+replicas" ) then
    set REPLICAS = $2
  endif
  set REPARGS = "$REPARGS $1"
  shift
end

set QUEUEARGS = "nodes=${NUMNODES}:ppn=32:xe"
set CUDA = ""

@ TOTALPROCS = $NUMNODES * $PROCSPERNODE

if ( $TOTALPROCS % $REPLICAS ) then
  echo "replicas must evenly divide total procs"
endif

switch ( $PROCSPERNODE )
case 1:
  set APRUNARGS = "-n ${TOTALPROCS} -r 1 -N 1 -d 31"
  #set NAMDARGS = "+ppn 31 +pemap 1-31 +commap 0"
  set NAMDARGS = "+ppn 30 +pemap 1-30 +commap 0"
  breaksw
case 2:
  set APRUNARGS = "-n ${TOTALPROCS} -N 2 -d 16"
  set NAMDARGS = "+ppn 15 +pemap 1-15,17-31 +commap 0,16"
  breaksw
case 3:
  set APRUNARGS = "-n ${TOTALPROCS} -N 3 -d 9"
  set NAMDARGS = "+ppn 8 +pemap 8-31 +commap 2,4,6"
  breaksw
case 4:
  set APRUNARGS = "-n ${TOTALPROCS} -N 4 -d 8"
  set NAMDARGS = "+ppn 7 +pemap 1-7,9-15,17-23,25-31 +commap 0,8,16,24"
  breaksw
case 31:
  set ARCH = "-ugni-BlueWaters"
  set APRUNARGS = "-n ${TOTALPROCS} -r 1 -N 31 -d 1"
  set NAMDARGS = "+pemap 0-30"
  breaksw
case 32:
  set ARCH = "-ugni-BlueWaters"
  set APRUNARGS = "-n ${TOTALPROCS} -N 32 -d 1"
  set NAMDARGS = "+pemap 0-31"
  breaksw
default:
  echo "Unrecognized processes per node count $PROCSPERNODE"
  exit -1
endsw

if ( $0:t =~ *cuda* ) then
  set ARCH = "-ugni-smp-BlueWaters"
  set CUDA = "-CUDA"
  set PROCSPERNODE = 1
  @ TOTALPROCS = $NUMNODES * $PROCSPERNODE
  set QUEUEARGS = "nodes=${NUMNODES}:ppn=16:xk"
  set APRUNARGS = "-n ${TOTALPROCS} -r 1 -N 1 -d 8"
  set NAMDARGS = "+ppn 7 +pemap 2-14:2 +commap 0"
endif

#if ( $QUEUE == test ) then
#  set TIME = 00:30:00
#  if ( `qstat -u $USER | egrep ' (R|Q) ' | grep xe_debug | wc -l` == 0 ) then
#    set QUEUE = feature=xe,qos=debug
#  else
#    set QUEUE = feature=xe
#  endif
  # if ( $NUMNODES <= 10 ) set QUEUE = scalability
  # if ( $NUMNODES <= 3 ) set QUEUE = development
#endif

if ( ! -f $CONFFILE) then
  echo "NAMD input file $CONFFILE does not exist!"
  exit -1
endif

if ( -f $LOGFILE) then
  echo "NAMD input file $LOGFILE already exists!"
  exit -1
endif

set RUNDIR = `pwd`

#if ( $RUNDIR !~ /lustre/* ) then
#  echo "This script must be run from within the /lustre filesystem,"
#  echo "which must contain all files read or written by NAMD."
#  exit -1
#endif

set MEMOPT = ""
if ( $0:t =~ *memopt* ) set MEMOPT = "-memopt"

echo "Submitting NAMD run to queue for $TIME..."
echo ""

set TMPFILE = /tmp/$USER.$$.pbs

cat > $TMPFILE << ENDINPUT
#!/bin/tcsh
#PBS -V
#PBS -j oe
#PBS -N NAMD
#PBS -q $QUEUE
#PBS -l walltime=$TIME,$QUEUEARGS

cd $RUNDIR

source ${MODULESHOME}/init/tcsh
module swap PrgEnv-cray PrgEnv-gnu
module load rca
module load craype-hugepages8M
module list

setenv HUGETLB_DEFAULT_PAGE_SIZE 8M
setenv HUGETLB_MORECORE no
setenv ATP_ENABLED 1
ulimit -c unlimited

echo PBS opts:  walltime=$TIME,$QUEUEARGS
echo aprun opts:  $APRUNARGS
echo NAMD opts:  $NAMDARGS $REPARGS
echo conf file:  $CONFFILE
echo log file:  $LOGFILE

#aprun $APRUNARGS /u/sciteam/jphillip/charm/gemini_gni-crayxe-hugepages-smp/tests/charm++/megatest/pgm >& ${LOGFILE}.test
#aprun $APRUNARGS /u/sciteam/jphillip/NAMD/NAMD_2.9_CRAY-XT${ARCH}${CUDA}${MEMOPT}/namd2 $NAMDARGS $REPARGS $CONFFILE >& ${LOGFILE}
aprun $APRUNARGS /u/sciteam/jphillip/NAMD_build.latest/NAMD_2.9_CRAY-XE${ARCH}${CUDA}${MEMOPT}/namd2 $NAMDARGS $REPARGS $CONFFILE >& ${LOGFILE}

ENDINPUT

cat $TMPFILE

qsub $TMPFILE

if ( $status ) then
echo ""
echo "Queue submission failed, please see errors above."
echo ""
else
echo ""
echo "NAMD is submitted to the queue, to check on your job use this command:"
echo ""
echo "    qstat -u $USER"
echo ""
echo "Your job is running if there is an R (running) in the S (state) column."
echo "You may monitor progress with this command:"
echo ""
echo "    tail -F $LOGFILE"
echo ""
echo "You will need to press Control-C to exit the tail command."
echo ""
endif

