Gamess (US) frequently asked questions Part 4: The rungms script

Third guest post by Kirill Berezovsky (Petrozadovsk State University).

Gamess (US) is run though a script provided in the installation package. This script MUST be adapted by the user. The rungms-script below has been adapted to be used with the Gamess (US) installation described in earlier posts. Happy computing :-)

RUNGMS script

#!/bin/csh

set SCR=/scr/$USER

set USERSCR=~$USER/scr

set GMSPATH=/usr/local/gamess

set JOB=$1

set VERNO=$2

if ($VERNO == cpu) set TARGET=mpi

if ($VERNO == gpu) set TARGET=ga

if (null$VERNO == null) set VERNO=cpu

set master=`hostname`

printf "\n * Started at:           `date`"

set DSK=`df -m $SCR | awk 'NR==2{print$4}'`

printf "\n * Available disk space: $DSK MB"

printf "\n * Temporary files in:   $SCR and in $USERSCR\n\n"

limit stacksize 8192

if ($JOB:r.inp == $JOB) set JOB=$JOB:r

if (-e $JOB.inp) then

   cp  $JOB.inp  $SCR/$JOB.F05

else

   echo "Input file $JOB.inp not found"

   exit 4

endif

source $GMSPATH/gms-files.csh

if (-e $HOME/.gmsrc) source $HOME/.gmsrc

set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=0 ' | wc -l`

if ($ngddi > 0) then

   set GDDIjob=true

   echo "This is a GDDI run, keeping various output files on local disks"

   set echo

   setenv  OUTPUT $SCR/$JOB.F06

   setenv   PUNCH $SCR/$JOB.F07

   unset echo

else

   set GDDIjob=false

endif

if ((-e $PUNCH) || (-e $MAKEFP) || (-e $TRAJECT) || (-e $RESTART) ) then

   echo "Please save, rename, or erase these files from a previous run:"

   echo "     $PUNCH,"

   echo "     $TRAJECT,"

   echo "     $RESTART, and/or"

   echo "     $MAKEFP,"

   echo "and then resubmit this computation."

   exit 4

endif

#----------------------------------------------------------------------

if ($TARGET == mpi) then

 set NCPUS=`grep cores /proc/cpuinfo | wc -l`

 echo " * CPU cores:   $NCPUS "

 echo " * GPU devices: not used"

 echo " "

 setenv TRAJECT $USERSCR/$JOB.trj

 setenv RESTART $USERSCR/$JOB.rst

 setenv INPUT $SCR/$JOB.F05

 setenv PUNCH $USERSCR/$JOB.dat

 if ( -e $TRAJECT ) rm $TRAJECT

 if ( -e  $PUNCH ) rm $PUNCH

 if ( -e  $RESTART ) rm $RESTART

 setenv LD_LIBRARY_PATH /opt/intel/impi/4.0.2.003/intel64/lib:$LD_LIBRARY_PATH

 set path= ( /opt/intel/impi/4.0.2.003/intel64/bin $path )

mpdboot

 mpiexec -n $NCPUS $GMSPATH/gamess.$VERNO.x

mpdallexit

 cp $PUNCH .

endif

#----------------------------------------------------------------------

if ($TARGET == ga) then

 set PPN=1

 set NCPUS=1

 @ NPROCS = $NCPUS

 setenv HOSTFILE $SCR/$JOB.nodes.mpd

 if (-e $HOSTFILE) rm $HOSTFILE

 touch $HOSTFILE

  

 echo `hostname` >> $HOSTFILE

 set NNODES=1

 setenv PROCFILE $SCR/$JOB.processes.mpd

 if (-e $PROCFILE) rm $PROCFILE

 touch $PROCFILE

 echo "-n $NPROCS -host `hostname` $GMSPATH/gamess.$VERNO.x" >> $PROCFILE

 set path=(/opt/intel/impi/4.0.2.003/intel64/bin $path)

   setenv I_MPI_WAIT_MODE enable

   setenv I_MPI_PIN disable

   setenv I_MPI_DEBUG 0

   setenv I_MPI_STATS 0

   setenv I_MPI_DEVICE sock

   setenv I_MPI_NETMASK ib0

 setenv LD_LIBRARY_PATH /opt/intel/impi/4.0.2.003/intel64/lib:$LD_LIBRARY_PATH

 setenv LD_LIBRARY_PATH /opt/intel/composerxe-2011.4.191/compiler/lib/intel64:$LD_LIBRARY_PATH

 setenv LD_LIBRARY_PATH /opt/intel/composer_xe_2013.0.079/mkl/lib/intel64:$LD_LIBRARY_PATH

 setenv MKL_SERIAL YES

 setenv MKL_NUM_THREADS 1

 setenv LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH

 setenv GMS_CCHEM '1'

 @ NUMGPU=1

 setenv CCHEM 'devices=0;memory=4g'

 echo " * CPU cores:   `grep cores /proc/cpuinfo | wc -l` "

 echo " * GPU devices: $NUMGPU (with settings: $CCHEM)"

 echo " "

 chdir $SCR

 set echo

 mpdboot --rsh=ssh -n $NNODES -f $HOSTFILE

  mpiexec -configfile $PROCFILE < /dev/null

 mpdallexit

 unset echo

 rm -f $PROCFILE

endif

#----------------------------------------------------------------------

echo ----- accounting info -----

if ($GDDIjob == true) cp $SCR/$JOB.F07 ~/scr/$JOB.dat

echo Files used on the master node $master were:

ls -lF $SCR/$JOB.*

rm -f  $SCR/$JOB.F*

if (-e $SCR/$JOB.V84)        mv $SCR/$JOB.V84     $USERSCR

if (-e $SCR/$JOB.V80)        rm -f $SCR/$JOB.V*

if (-e $SCR/$JOB.TEMP02)     rm -f $SCR/$JOB.TEMP*

if (-e $SCR/$JOB.orb)        mv $SCR/$JOB.orb     $USERSCR

if (-e $SCR/$JOB.vec)        mv $SCR/$JOB.vec     $USERSCR

if (-e $SCR/$JOB.mol)        mv $SCR/$JOB.mol     $USERSCR

if (-e $SCR/$JOB.molf)       mv $SCR/$JOB.molf    $USERSCR

if (-e $SCR/$JOB.mkl)        mv $SCR/$JOB.mkl     $USERSCR

if (-e $SCR/$JOB.xyz)        mv $SCR/$JOB.xyz     $USERSCR

ls $SCR/${JOB}-*.cube > $SCR/${JOB}.lis

if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.cube $USERSCR

rm -f $SCR/${JOB}.lis

ls $SCR/${JOB}-*.grd > $SCR/${JOB}.lis

if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.grd $USERSCR

rm -f $SCR/${JOB}.lis

ls $SCR/${JOB}-*.csv > $SCR/${JOB}.lis

if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.csv $USERSCR

rm -f $SCR/${JOB}.lis

if ($TARGET == mpi) then

   set nnodes=`wc -l $HOSTFILE`

   set nnodes=$nnodes[1]

   @ n=1

   set master=`hostname`

   set master=$master:r

   while ($n <= $nnodes)

      set host=`sed -n -e "$n p" $HOSTFILE`

      set host=$host[1]

      if ($host != $master) then

         echo Files used on node $host were:

         ssh $host -l $USER "ls -l $SCR/$JOB.*"

         ssh $host -l $USER "rm -f $SCR/$JOB.*"

      endif

      @ n++

   end

   rm -f $HOSTFILE

   if ($?I_MPI_STATS) then

      if ($I_MPI_STATS > 0) mv $SCR/stats.txt ~/$JOB.$NCPUS.stats

   endif

endif

date

time

exit