Wednesday, June 26, 2013

Gamess (US) frequently asked questions Part 4: The rungms script

Third guest post by Kirill Berezovsky (Petrozadovsk State University).

Gamess (US) is run though a script provided in the installation package. This script MUST be adapted by the user. The rungms-script below has been adapted to be used with the Gamess (US) installation described in earlier posts. Happy computing :-)

RUNGMS script


set SCR=/scr/$USER
set USERSCR=~$USER/scr
set GMSPATH=/usr/local/gamess

set JOB=$1
set VERNO=$2

if ($VERNO == cpu) set TARGET=mpi
if ($VERNO == gpu) set TARGET=ga
if (null$VERNO == null) set VERNO=cpu

set master=`hostname`
printf "\n * Started at:           `date`"

set DSK=`df -m $SCR | awk 'NR==2{print$4}'`
printf "\n * Available disk space: $DSK MB"

printf "\n * Temporary files in:   $SCR and in $USERSCR\n\n"

limit stacksize 8192

if ($JOB:r.inp == $JOB) set JOB=$JOB:r
if (-e $JOB.inp) then
   cp  $JOB.inp  $SCR/$JOB.F05
   echo "Input file $JOB.inp not found"
   exit 4

source $GMSPATH/gms-files.csh
if (-e $HOME/.gmsrc) source $HOME/.gmsrc

set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=0 ' | wc -l`
if ($ngddi > 0) then
   set GDDIjob=true
   echo "This is a GDDI run, keeping various output files on local disks"
   set echo
   setenv  OUTPUT $SCR/$JOB.F06
   setenv   PUNCH $SCR/$JOB.F07
   unset echo
   set GDDIjob=false

if ((-e $PUNCH) || (-e $MAKEFP) || (-e $TRAJECT) || (-e $RESTART) ) then
   echo "Please save, rename, or erase these files from a previous run:"
   echo "     $PUNCH,"
   echo "     $TRAJECT,"
   echo "     $RESTART, and/or"
   echo "     $MAKEFP,"
   echo "and then resubmit this computation."
   exit 4


if ($TARGET == mpi) then

 set NCPUS=`grep cores /proc/cpuinfo | wc -l`

 echo " * CPU cores:   $NCPUS "
 echo " * GPU devices: not used"
 echo " "

 setenv INPUT $SCR/$JOB.F05
 setenv PUNCH $USERSCR/$JOB.dat

 if ( -e $TRAJECT ) rm $TRAJECT
 if ( -e  $PUNCH ) rm $PUNCH
 if ( -e  $RESTART ) rm $RESTART

 setenv LD_LIBRARY_PATH /opt/intel/impi/$LD_LIBRARY_PATH
 set path= ( /opt/intel/impi/ $path )
 mpiexec -n $NCPUS $GMSPATH/gamess.$VERNO.x
 cp $PUNCH .


if ($TARGET == ga) then
 set PPN=1
 set NCPUS=1


 setenv HOSTFILE $SCR/$JOB.nodes.mpd
 touch $HOSTFILE
 echo `hostname` >> $HOSTFILE
 set NNODES=1

 setenv PROCFILE $SCR/$JOB.processes.mpd
 touch $PROCFILE

 echo "-n $NPROCS -host `hostname` $GMSPATH/gamess.$VERNO.x" >> $PROCFILE

 set path=(/opt/intel/impi/ $path)

   setenv I_MPI_WAIT_MODE enable
   setenv I_MPI_PIN disable
   setenv I_MPI_DEBUG 0
   setenv I_MPI_STATS 0
   setenv I_MPI_DEVICE sock
   setenv I_MPI_NETMASK ib0

 setenv LD_LIBRARY_PATH /opt/intel/impi/$LD_LIBRARY_PATH
 setenv LD_LIBRARY_PATH /opt/intel/composerxe-2011.4.191/compiler/lib/intel64:$LD_LIBRARY_PATH
 setenv LD_LIBRARY_PATH /opt/intel/composer_xe_2013.0.079/mkl/lib/intel64:$LD_LIBRARY_PATH


 setenv LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH

 setenv GMS_CCHEM '1'

 setenv CCHEM 'devices=0;memory=4g'

 echo " * CPU cores:   `grep cores /proc/cpuinfo | wc -l` "
 echo " * GPU devices: $NUMGPU (with settings: $CCHEM)"
 echo " "

 chdir $SCR

 set echo
 mpdboot --rsh=ssh -n $NNODES -f $HOSTFILE
  mpiexec -configfile $PROCFILE < /dev/null
 unset echo

 rm -f $PROCFILE


echo ----- accounting info -----

if ($GDDIjob == true) cp $SCR/$JOB.F07 ~/scr/$JOB.dat

echo Files used on the master node $master were:
ls -lF $SCR/$JOB.*
rm -f  $SCR/$JOB.F*

if (-e $SCR/$JOB.V84)        mv $SCR/$JOB.V84     $USERSCR
if (-e $SCR/$JOB.V80)        rm -f $SCR/$JOB.V*
if (-e $SCR/$JOB.TEMP02)     rm -f $SCR/$JOB.TEMP*
if (-e $SCR/$JOB.orb)        mv $SCR/$JOB.orb     $USERSCR
if (-e $SCR/$JOB.vec)        mv $SCR/$JOB.vec     $USERSCR
if (-e $SCR/$JOB.mol)        mv $SCR/$JOB.mol     $USERSCR
if (-e $SCR/$JOB.molf)       mv $SCR/$JOB.molf    $USERSCR
if (-e $SCR/$JOB.mkl)        mv $SCR/$JOB.mkl     $USERSCR
if (-e $SCR/$        mv $SCR/$     $USERSCR
ls $SCR/${JOB}-*.cube > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.cube $USERSCR
rm -f $SCR/${JOB}.lis
ls $SCR/${JOB}-*.grd > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.grd $USERSCR
rm -f $SCR/${JOB}.lis
ls $SCR/${JOB}-*.csv > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.csv $USERSCR
rm -f $SCR/${JOB}.lis

if ($TARGET == mpi) then
   set nnodes=`wc -l $HOSTFILE`
   set nnodes=$nnodes[1]
   @ n=1
   set master=`hostname`
   set master=$master:r
   while ($n <= $nnodes)
      set host=`sed -n -e "$n p" $HOSTFILE`
      set host=$host[1]
      if ($host != $master) then
         echo Files used on node $host were:
         ssh $host -l $USER "ls -l $SCR/$JOB.*"
         ssh $host -l $USER "rm -f $SCR/$JOB.*"
      @ n++
   rm -f $HOSTFILE
   if ($?I_MPI_STATS) then
      if ($I_MPI_STATS > 0) mv $SCR/stats.txt ~/$JOB.$NCPUS.stats


Gamess (US) frequently asked questions Part 3: Compiling Gamess (US) in a GPU-enabled environment

A second guest post by Kirill Berezovsky (Petrozadovsk State University)

Gamess (US) may use GPUs for MP2 andCCSD(T) jobs. If you will use NVIDIA GPU, then additionally install these packages:

1.     NVIDIA CUDA 4.1

2.     HDF5 1.8.7

But, first install NVIDIA video driver by terminal (it should be 304.64 and upper):

1.     Get driver:          
This site allows the manual download of the required drivers, but may also detect automatically the drivers your system needs. Neat ;-)

2.     Run in terminal:
o    echo “blacklist nvidiafb” >> /etc/modprobe.d/blacklist.conf
o    echo “blacklist nouveau” >> /etc/modprobe.d/blacklist.conf
o    echo “blacklist rivafb” >> /etc/modprobe.d/blacklist.conf
o    echo “blacklist rivatv” >> /etc/modprobe.d/blacklist.conf
o    echo “blacklist vga16fb” >> /etc/modprobe.d/blacklist.conf
o    echo “options nouveau modeset=0” >> /etc/modprobe.d/blacklist.conf

3.     Next remove all pre-installed nvidia-packages (in terminal):
o    apt-get --purge remove nvidia*

4.     Restart your machine and press Ctrl-Alt-F1 (or the equivalent key sequence for your machine) and enter as root. Kill GUI by:
o    killall gdm3       (for Debian)
o    killall gdm         (for Ubuntu 10.xx)
o    killall lightgdm   (for Ubuntu 11.xx)
o    killall lighdm     (for Ubuntu 12.xx)

5.     Go to folder, where NVIDIA-driver places and:
o    chmod +x
o    ./

6.     When you install driver, restart your machine, and install NVIDIA CUDA.

7.     HDF5 compiled by:
o    tar xvf hdf5-1.8.7.tar.bz2
o    cd hdf5-1.8.7
o    ./configure --prefix=/usr/local/hdf5 --enable-fortran --enable-parallel FC=mpiifort
o    make
o    make install

8.     Add in ~/.bashrc new variables:

export PATH=/usr/local/cuda/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/usr/local/cuda/lib:$LD_LIBRARY_PATH

9.     Re-run GAMESS (US) ./config script again, but at this time answer “yes” for LIBCCHEM-question, and choose paths for CUDA and HDF5.

10.  Go to libcchem folder and run script ./build-cchem.

11.  Wait…

12.  Go upper folder and link new executable of GAMESS (US):
o    ./lked gamess gpu

13.  Great! Now you can run MP2 and CCSD(T) jobs on gpu by command using my script:
gamess Inputfile.input gpu