#!/bin/bash
#PBS -N BGG_harm_disp
#PBS -q B-2
#PBS -j oe
#PBS -l nodes=4:ppn=16

ulimit -s unlimited

#***********************************************************************************
NODELIST=`cat ${PBS_NODEFILE} | sort | uniq`
for node in ${NODELIST}
do
    FLAG_MPDPY=`ssh ${node} /bin/ps ux | /bin/grep python | /bin/grep mpd.py`

    if [ -n "${FLAG_MPDPY}" ]; then
	echo "mpd.py is already running on host" ${node}

	ssh ${node} mpdallexit
	FLAG_MPDPY=`ssh ${node} /bin/ps ux | /bin/grep python | /bin/grep mpd.py`
	if [ -z "${FLAG_MPDPY}" ]; then
	    echo "mpd.py on node" ${node} "was successfully killed."
	else 
	    echo "could not terminate mpd.py on node" ${node} 
	fi
    fi
done
#***********************************************************************************

NUM_NODES=4
NUM_CORES=16
NUM_THREADS=1
NUM_PROCS=`expr $NUM_NODES \* $NUM_CORES / $NUM_THREADS`

export I_MPI_PERHOST=`expr $NUM_CORES / $NUM_THREADS`
export I_MPI_FABRICS=shm:ofa
export I_MPI_PIN_DOMAIN=omp
export I_MPI_PIN_CELL=core

export OMP_NUM_THREADS=$NUM_THREADS
export KMP_STACKSIZE=16m
export KMP_AFFINITY=compact

mpdboot -n $NUM_NODES -f ${PBS_NODEFILE} 

cd ${PBS_O_WORKDIR}

#Magnitude of displacements in Angstrom
disp_Angstrom=0.04

#cell parameter in Angstrom
cella=10.954

disp=`echo "scale=20; print $disp_Angstrom/$cella" | bc`

filename="harm_pattern"

BUFIFS=$IFS
IFS=
exec 3< $filename
while read FL 0<&3
do
    atom1=`echo ${FL} | awk '{print $1}'`

    for sign1 in + -
    do

	symbol1=${sign1}${atom1}

	OUTFILE=OUTCAR.${disp_Angstrom}_${symbol1}_${symbol2}
        OUTxml=vasprun_${disp_Angstrom}_${symbol1}_${symbol2}.xml

        if [ -e ${OUTFILE} -o -e ${OUTxml} ]; then
	    continue
        fi


#Add the header to POSCAR
	cat > POSCAR << EOF
BGG_${disp_Angstrom}_${symbol1}
${cella}
1.0 0.0 0.0
0.0 1.0 0.0
0.0 0.0 1.0
Ba Ga Ge
8 16 30
Direct
EOF

#Add the displaced coordinate to POSCAR
	gendispcoord.py POSCAR0 $disp ${symbol1} >> POSCAR

#Exec calculation
	mpiexec -np $NUM_PROCS ~/bin/vasp-half

#Rename files
	mv OUTCAR OUTCAR.${disp_Angstrom}_${symbol1}
	mv vasprun.xml vasprun_${disp_Angstrom}_${symbol1}.xml
    done
done
exec 3<&-

IFS=$BUFIFS

mpdallexit

