#--------------- g03job sample script --------
# Will work on any of the arches clusters
#
#PBS -S /bin/csh
#PBS -l walltime=02:00:00,nodes=2:ppn=2
#PBS -N test
#
# Insert proper node and walltime specifications above making sure that on
# arches that ppn is set to 2 (marchingmen, tunnelarch, delicatearch, part of 
# landscapearch) or 4 (some of landscapearch, sanddunearch)
# and that this value agrees with the  value of %nprocs in the input file
# the nodes value should agree with the %nprocl in the input file 
#
# Other node specifications such as memory can also be added to above, as well
# as qos specifications.
# Add job name on PBS -N line above; this will become name for stderr and stdout, 
# with extensions e#### and o#### respectively, where #### is the job number
#
# Put in proper WORKDIR, FILENAME in the first two lines below, respectively
#
# Set SCRFLAG to one of the following:
#	TMP for use of tmp space local to nodes (small space and automatically scrubbed after each job)
#	MM for serial scratch for MM only
#	DA for serial scratch for DA only
#	SERIALOLD for serial-old on all of arches
#	SERIAL for serial scratch on all of arches
#
# Set  NODES in a manner consistent to the PBS -l line above
#
# Put in USER ID for use in scratch directory path
#
# Also make sure %nprocs and/or %nprocl and %mem are set properly in the com file!
# Allow at least 64MB of the memory available per node for the OS
#**************************
# RECOMMENDATION: no more than 1GB on marchingmen, delicatearch
#                 no more than 2GB on tunnelarch
#***************************
#
# On multiple run nodes set %nprocl to the number of NODES specified
# 
setenv WORKDIR $HOME/project1
setenv FILENAME test
setenv SCRFLAG MM
setenv NODES 2
#
# nothing should need changed below here to run unless you do not
# want to use the default version of Gaussian; in this case the
# g03root path will need to be changed
#
cd $WORKDIR

setenv g03root /uufs/arches/sys/g03
set path = ($path $g03root/g03/linda7.1/opteron-linux-I8/bin)

setenv MP_NEWJOB yes
setenv LINDA_CLC network
setenv LINDA_FLC network

cat $PBS_NODEFILE | uniq > $WORKDIR/Gau.hosts
set NODELIST = `xargs echo < Gau.hosts`
cat Gau.hosts | xargs >tsnet.nodes
echo $NODELIST

if ("$SCRFLAG" == "TMP") then
  setenv GAUSS_SCRDIR  /tmp/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
  foreach HOST (`cat $PBS_NODEFILE`)
        rsh $HOST mkdir -p /tmp/$USER/$PBS_JOBID 
  end
endif

if ("$SCRFLAG" == "SERIALOLD") then
  setenv GAUSS_SCRDIR  /scratch/serial-old/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/serial-old/$USER/$PBS_JOBID
   cd /scratch/serial-old/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

if ("$SCRFLAG" == "SERIAL") then
  setenv GAUSS_SCRDIR  /scratch/serial/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/serial/$USER/$PBS_JOBID
   cd /scratch/serial/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

if ("$SCRFLAG" == "MM") then
  setenv GAUSS_SCRDIR  /scratch/mm/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/mm/$USER/$PBS_JOBID
   cd /scratch/mm/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

if ("$SCRFLAG" == "DA") then
  setenv GAUSS_SCRDIR  /scratch/da/$USER/$PBS_JOBID
  source $g03root/g03/bsd/g03.login
 mkdir -p /scratch/da/$USER/$PBS_JOBID
   cd /scratch/da/$USER/$PBS_JOBID
   cp $WORKDIR/$FILENAME.com .
   cp $WORKDIR/$FILENAME.chk .
endif

echo $GAUSS_SCRDIR
setenv GAUSS_LFLAGS "-nodelist '$NODELIST'"
echo $GAUSS_LFLAGS

# Run the program
if ("$NODES" == "1") then
g03 $FILENAME.com
else
g03l $FILENAME.com
endif

# Clean up scratch space

if ("$SCRFLAG" == "SERIALOLD") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/serial-old/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "SERIAL") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/serial/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "MM") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/mm/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "DA") then
  cp *.log $WORKDIR/.
  cp *.chk $WORKDIR/.
  cp Test.FChk $WORKDIR/$FILENAME.FChk
  cd ..
  rm -r /scratch/da/$USER/$PBS_JOBID
endif
if ("$SCRFLAG" == "TMP") then
  foreach HOST (`cat $PBS_NODEFILE`)
        rsh $HOST rm -rf /tmp/$USER/$PBS_JOBID 
  end
endif

#----- End of g03job ---------