#! /bin/sh
#
# Default values
np=1
maxtime=30
infiles=""
outfiles=""
verbose="no"
shared="no"
local="no"
showtimes="yes"
ctransin="parin"
ctransout="parout"
showoutput="no"
keeptmp="no"
#
# Process arguments
#
while [ $# -gt 0 ] ; do
    arg=$1
    shift
    case $arg in 
    -np)
    np=$1
    shift
    ;;
    -keeptmp)
    keeptmp="yes"
    ;;
    -local)
    local="yes"
    ;;
    -shared)
    shared="yes"
    ;;
    -maxtime)
    maxtime=$1
    shift
    ;;
    -showtimesoff)
    showtimes="no"
    ;;
    -showoutput)
    showoutput="yes"
    ;;
    -infile)
    infiles="$infiles $1"
    shift
    ;;
    -outfile)
    shift
    outfiles="$outfiles $1"
    shift
    ;;
    -stdin)
	# Name of a file to use for standard input.
	stdinfile="$1"
	shift
	;;
    -stdout)
        # Name of a file to use for standard output.
        stdoutfile="$1"
        shift
        ;;
    -stderr)
        # Name of a file to use for standard error.
        stderrfile="$1"
        shift
        ;;
    -echo)
    set -x
    ;;
    -u|-usage|-help)
    echo "$0       [-np n] [-maxtime min] [-showoutput ] "
    echo "         [-showtimesoff] [-shared] prog [ progs args ]"
    exit 0;
    ;;
    *)
    if [ -x $arg ] ; then
        programname=$arg
        progargs=$@
    else
        echo "Unrecognized option $arg"
	exit 1
    fi
    break
    esac
done
#
# Now process program options looking for possible input files
# stage the file directly into the rundir directory so we need to generate new
# command line arguments that strip the path form the filename
#
newprogargs=""
shortinfiles=""
while [ $# -gt 0 ] ; do
    arg=$1
    shift
    case $arg in 
    -start_in_debugger|-on_error_attach_debugger) 
      shared="yes"
      local="no"
      newprogargs="$newprogargs $arg"
    ;;
    -f|-f0|-f1|-options_file) 
      infiles="$infiles $1"
      shortinfiles="$shortinfiles `basename $1`"
      newprogargs="$newprogargs $arg `basename $1`"
      if [ -e "$1.info" ] ; then
        infiles="$infiles $1.info"
      fi
      shift
    ;;
    *)
      if [ -e $arg ] ; then 
        infiles="$infiles $arg"
        shortinfiles="$shortinfiles `basename $arg`"
        newprogargs="$newprogargs `basename $arg`"
        if [ -e "$arg.info" ] ; then
          infiles="$infiles $arg.info"
        fi
      else 
        newprogargs="$newprogargs $arg"
      fi
    ;; 
  esac
done
if [ $local = "no" ] ; then
  if [ $showtimes = "yes" ] ; then
    if [ "$infiles" != "" ] ; then echo "Staging files: $infiles" ; fi
  fi
fi
#
jobdir=$HOME/myjobs
if [ ! -d $jobdir ] ; then mkdir $jobdir ; fi
rundir=$jobdir/$$
mkdir $rundir
#
# copy program, scripts etc into run directory
#
cp $programname $rundir
if [ "$infiles" != "" ] ; then cp $infiles $rundir; fi
cp ${PETSC_DIR}/bin/urlget $rundir
cp ${PETSC_DIR}/bin/urlget.py $rundir
cp ${PETSC_DIR}/bin/ $rundir
#
# Check if this should be a myrinet job:
#
hasgm="`nm $programname | egrep 'T gm_init$'`"
if [ -n "$hasgm" ] ; then
  usemyrinet="yes"
  myrinetattribute=":myrinet"
else
  usemyrinet="no"
  myrinetattribute=" "
fi
#
# strip path from program name
#
programbase=`basename $programname`
#
#
#
# Create the PBS script
#
    cat >./.mpiexec$$ <<EOF
#! /bin/bash
#PBS -S /bin/sh
#PBS -l nodes=${np}${myrinetattribute}
#PBS -l walltime=0:${maxtime}:00
##ctrans $ctransout $rundir
#
if [ $local = "no" ] ; then
  if [ $showtimes = "yes" ] ; then
    echo -n "Staging files to nodes " 
    date
  fi
  /my/bin/ctrans -t $ctransin -s $rundir -d $rundir -nf \$PBS_NODEFILE 
fi
if [ $showtimes = "yes" ] ; then
  echo -n "Starting mpi program "
  date
fi

PATH=$PATH
export PATH
DISPLAY=$DISPLAY
export DISPLAY
cd $rundir
if [ $showoutput = "yes" ] ; then
  pbs_job_num=\`expr \$PBS_JOBID : '\([0-9]*\).*'\`
  fileo=/var/spool/pbs/spool/\$pbs_job_num.ccsche.OU
  xterm -e tail -f \$fileo  &
fi

if [ $usemyrinet = "yes" ] ; then
  MPIEXEC="/soft/apps/packages/mpich-gm-1.2.4..8a-gm-1.5.2.1-gcc/bin/mpirun"
else
  MPIEXEC="/soft/apps/packages/mpich-1.2.4-gcc/bin/mpirun"
fi

MPIEXEC_CMD="\$MPIEXEC -np $np -machinefile \$PBS_NODEFILE $programbase -display $DISPLAY $newprogargs"

if [ $shared = "yes" ]; then
  rsh \$masternode "cd $rundir; \$MPIEXEC_CMD"
else
  \$MPIEXEC_CMD
fi
  
cat \$PBS_NODEFILE | /my/bin/pdsh -w - "cd $rundir ; rm -rf $programbase urlget  urlget.py $shortinfiles ; rm -rf PI*" > /dev/null 2>&1


if [ $showtimes = "yes" ] ; then
  echo -n "Bringing data files back "
  date
fi
/my/bin/ctrans -t $ctransout -s $rundir -d $rundir -nf \$PBS_NODEFILE
if [ $showtimes = "yes" ] ; then
  echo -n "Done "
  date
fi
#
exit 0;
EOF
#
#  Don't use batch system, just run script directly
# 
if [ $shared = "yes" ] ; then
  head -$np ${PETSC_DIR}/bin/hostnames.chiba > ./.pbsnodefile.$$
  export masternode=`tail -1 ./.pbsnodefile.$$`
  export PBS_NODEFILE=.pbsnodefile.$$
  cp  ./.pbsnodefile.$$ $rundir
  chmod u+x ./.mpiexec$$
  ./.mpiexec$$
#
#  Run directly on the frontend
#
elif [ $local = "yes" ] ; then
  echo "localhost" > ./.pbsnodefile.$$
  echo "localhost" >> ./.pbsnodefile.$$
  echo "localhost" >> ./.pbsnodefile.$$
  echo "localhost" >> ./.pbsnodefile.$$
  PBS_NODEFILE=`pwd`/.pbsnodefile.$$
  export PBS_NODEFILE
  chmod u+x ./.mpiexec$$
  ./.mpiexec$$
else 
#
#  submit to batch system
#
  pbs_job_id=`qsub ./.mpiexec$$`
  pbs_job_num=`expr $pbs_job_id : '\([0-9]*\).*'`
#
#
#  Handle control c from user
#
  trap "qdel $pbs_job_id ; echo Killing job; exit" 2
#
  if [ $showtimes = "yes" ] ; then
    echo -n "PBS Job $pbs_job_id Submitted to queue " 
    date
  fi

#  Spin until qstat returns a non-zero return code
#
  status="Q"
  while [ "$status" = "Q" ] ; do
    sleep 5
    lstatus=`qstat $pbs_job_id 2> /dev/null`
    status=`echo $lstatus | tr -s "" " " | cut -f19 -d" "`
    if [ $showtimes = "yes" ] ; then
      echo -n "."
    fi
  done
  if [ $showtimes = "yes" ] ; then
    echo " "
  fi
# 
  if [ $showtimes = "yes" ] ; then
    echo -n "PBS Job $pbs_job_id Running "
    date
  fi
#
#
  status="R"
  while [ "$status" = "R" ] ; do
    sleep 2
    lstatus=`qstat $pbs_job_id 2> /dev/null`
    status=`echo $lstatus | tr -s "" " " | cut -f19 -d" "`
    if [ $showtimes = "yes" ] ; then
      echo -n "."
    fi
  done
  if [ $showtimes = "yes" ] ; then
    echo " "
  fi
#
  if [ $showtimes = "yes" ] ; then
    echo -n "PBS Job $pbs_job_id Job no longer running "
    date
  fi

#
  outstat=""
  while [ "$outstat" != "Unknown Job" ] ; do
    sleep 2
    outstat=`qstat $pbs_job_id 2>&1 | sed -e 's/^.*Unknown Job.*$/Unknown Job/'`
    if [ $showtimes = "yes" ] ; then
      echo -n "."
    fi
  done
  if [ $showtimes = "yes" ] ; then
    echo " "
  fi
#
  if [ $showtimes = "yes" ] ; then
    echo -n "PBS Job $pbs_job_id No longer existing "
    date
  fi
#
#  Move the output to the stdout/err of the process running this script
#
  if [ -s .mpiexec$$.o$pbs_job_num ] ; then
    if [ -n "$stdoutfile" ] ; then
      cat .mpiexec$$.o$pbs_job_num > $stdoutfile
    else
      cat .mpiexec$$.o$pbs_job_num 
    fi
  fi
  if [ -s .mpiexec$$.e$pbs_job_num ] ; then
    if [ -n "$stderrfile" ] ; then
        cat .mpiexec$$.e$pbs_job_num >$stderrfile
    else
        cat .mpiexec$$.e$pbs_job_num >&2
    fi
  fi

fi
#
# decide if we should make an output directory (was there output?)
#
if [ $local = "no" ] ; then
  pushd $rundir > /dev/null
  rm -rf $programbase $shortinfiles urlget  urlget.py
  rm -rf [0-9]*
  popd > /dev/null
  if [ "`ls $rundir/*`" != "" ] ; then
    cp -rf $rundir ./output.$$ 
    cd output.$$
    mv */*.[0-9]* . > /dev/null 2>&1
    if [ "`ls ./ccn[0-9]*/* 2> /dev/null`" = "" ] ; then
      rm -rf ccn[0-9]*
    fi
    cd ..
    if [ "`ls ./output.$$/* 2> /dev/null`" = "" ] ; then
      rm -rf output.$$
    else 
      if [ $showtimes = "yes" ] ; then
        echo "Output files placed into " output.$$
      fi
    fi
  fi
fi

#
#  Remove all files mpiexec.chiba has created
#
if [ $keeptmp != "yes" ] ; then
  rm -rf .mpiexec$$* 
  rm -f .pbsnodefile*
  rm -rf $rundir
fi
