#--------------- g03job sample script --------
# Will work on any of the arches clusters and on updraft
#
#PBS -S /bin/csh
#PBS -l walltime=24:00:00,nodes=16:ppn=8
#PBS -N test
#
# Insert proper node and walltime specifications above making sure that on
# arches that ppn is set to 2 (MM, TA, DA), 4 (SDA), or 8 (Updraft)
# and that this value agrees with the  value of %nprocs in the input file
# the nodes value should agree with the %nprocl in the input file 
#
# Other node specifications such as memory can also be added to above, as well
# as qos specifications.
# Add job name on PBS -N line above; this will become name for stderr and stdout, 
# with extensions e#### and o#### respectively, where #### is the job number
#
# Put in proper WORKDIR, FILENAME in the first two lines below, respectively
#
# Set SCRFLAG to one of the following:
#       TMP for use of tmp space local to nodes (make sure that /tmp is large enough!)
#       MM for serial scratch for MM only
#       DA for serial scratch for DA only
#       GENERAL for the serial scratch on updraft
#       SERIAL for serial scratch on all arches and updraft
#
# Set  NODES in a manner consistent to the PBS -l line above
#
# Put in USER ID for use in scratch directory path
#
# Also make sure %nprocs and/or %nprocl and %mem are set properly in the com file!
# Allow at least 64MB of the memory available per node for the OS
# I have not found any limitations of memory use outside of that available on nodes
# On multiple run nodes set %nprocl to the number of NODES specified
# 
setenv WORKDIR $HOME/grissom
setenv FILENAME molecule1-p
setenv SCRFLAG GENERAL
setenv NODES 8
#
# nothing should need changed below here to run unless you do not
# want to use the default version of Gaussian; in this case the
# g03root path will need to be changed
#
cd $WORKDIR

if ($UUFSCELL == "updraft.arches") then
setenv g03root /uufs/chpc.utah.edu/sys/pkg/gaussian03/E.01-EMT64
set path = ($path $g03root/g03/linda7.1/opteron-linux-I8/bin)
else
setenv g03root /uufs/chpc.utah.edu/sys/pkg/gaussian03/std
set path = ($path $g03root/g03/linda7.1/opteron-linux-I8/bin)
endif
setenv MP_NEWJOB yes
setenv LINDA_CLC network
setenv LINDA_FLC network

cat $PBS_NODEFILE | uniq > $WORKDIR/Gau.hosts
set NODELIST = `xargs echo < Gau.hosts`
cat Gau.hosts | xargs >tsnet.nodes
echo $NODELIST


if ("$SCRFLAG" == "TMP") then
  setenv GAUSS_SCRDIR  /tmp/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
  foreach HOST (`cat $PBS_NODEFILE`)
        rsh $HOST mkdir -p /tmp/$USER/$PBS_JOBID 
  end
endif

if ("$SCRFLAG" == "SERIAL") then
  setenv GAUSS_SCRDIR  /scratch/serial/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/serial/$USER/$PBS_JOBID
   cd /scratch/serial/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

if ("$SCRFLAG" == "GENERAL") then
  setenv GAUSS_SCRDIR  /scratch/general/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/general/$USER/$PBS_JOBID
   cd /scratch/general/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

if ("$SCRFLAG" == "MM") then
  setenv GAUSS_SCRDIR  /scratch/mm/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/mm/$USER/$PBS_JOBID
   cd /scratch/mm/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

if ("$SCRFLAG" == "DA") then
  setenv GAUSS_SCRDIR  /scratch/da/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/da/$USER/$PBS_JOBID
   cd /scratch/da/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

echo $GAUSS_SCRDIR
setenv GAUSS_LFLAGS "-nodelist '$NODELIST'"
echo $GAUSS_LFLAGS

# Run the program
if ("$NODES" == "1") then
g03 $FILENAME.com
else
g03l $FILENAME.com
endif

# Clean up scratch space

if ("$SCRFLAG" == "SERIAL") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/serial/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "GENERAL") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/general/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "MM") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/mm/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "DA") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/da/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "TMP") then
  foreach HOST (`cat $PBS_NODEFILE`)
        rsh $HOST rm -rf /tmp/$USER/$PBS_JOBID 
  end
endif

#----- End of g03job ---------