#!/bin/bash
#PBS -N GG_mode_LA_9
#PBS -q A-2
#PBS -j oe
#PBS -l nodes=1:ppn=16

ulimit -s unlimited

#***********************************************************************************
NODELIST=`cat ${PBS_NODEFILE} | sort | uniq`
echo $NODELIST
for node in ${NODELIST}
do
    FLAG_MPDPY=`ssh ${node} /bin/ps ux | /bin/grep python | /bin/grep mpd.py`

    if [ -n "${FLAG_MPDPY}" ]; then
	echo "mpd.py is already running on host" ${node}

	ssh ${node} mpdallexit
	FLAG_MPDPY=`ssh ${node} /bin/ps ux | /bin/grep python | /bin/grep mpd.py`
	if [ -z "${FLAG_MPDPY}" ]; then
	    echo "mpd.py on node" ${node} "was successfully killed."
	else 
	    echo "could not terminate mpd.py on node" ${node} 
	fi
    fi
done
#***********************************************************************************

NUM_NODES=1
NUM_CORES=16
NUM_THREADS=16
NUM_PROCS=`expr $NUM_NODES \* $NUM_CORES / $NUM_THREADS`

export I_MPI_PERHOST=`expr $NUM_CORES / $NUM_THREADS`
export I_MPI_FABRICS=shm:ofa
export I_MPI_PIN_DOMAIN=omp
export I_MPI_PIN_CELL=core

export OMP_NUM_THREADS=$NUM_THREADS
export KMP_STACKSIZE=8m
export KMP_AFFINITY=compact

mpdboot -n $NUM_NODES -f ${PBS_NODEFILE} 

cd ${PBS_O_WORKDIR}
mpirun -np $NUM_PROCS ~/Work/alm/anphon/anphon input_BTE9.txt > GG_9.log
mpdallexit
