### Job Name
#PBS -N mpi_bench
### Project code
#PBS -A MPI_Bench
#PBS -l walltime=00:10:00
#PBS -q xeon1600
###PBS -q xeon1800s
###PBS -q xeon1800w
###PBS -q xeon1700s
###PBS -q xeon1700w
### Merge output and error files
#PBS -j oe
#PBS -k eod
### Select 2 nodes with 2 CPUs 
###PBS -lselect=1:ncpus=104:mpiprocs=104:switch=10gBE,place=scatter:excl
#PBS -lselect=2:ncpus=40:mpiprocs=40
###PBS -lselect=1:ncpus=100:mpiprocs=100
###PBS -lselect=1:ncpus=50:mpiprocs=50
###PBS -lselect=1:ncpus=200:mpiprocs=200
### Send email on abort, begin and end
#PBS -m abe
### Specify mail recipient
#PBS -M john.fowler@xd-lab.net 
# /utils/opt/intel/impi/2019.9.304/bin/compilervars.sh  -arch intel64 -platform linux
. /utils/opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh
# . /utils/opt/intel/impi/2019.5.281/intel64/bin/mpivars.sh
. /utils/opt/intel/mkl/bin/mklvars.sh  intel64
. /utils/spack/share/spack/setup-env.sh
source /utils/opt/intel/impi/2021.1.1/setvars.sh
# PATH=/utils/opt/intel/impi/2019.9.304/impi/2019.9.304/intel64/bin/:$PATH
export PBS_NODEFILE=cluster.txt
export PBS_MPI_DEBUG=True 
export MPI_USE_IB=False
export I_MPI_FABRICS=ofi	
# export I_MPI_SHM=auto
export I_MPI_STATS=ipm
export I_MPI_NETMASK=eth
export FI_TCP_IFACE=eno1
export I_MPI_HYDRA_IFACE=eno1
echo "Start" 
cd ~/tpbs
date
ls 
printenv | grep PBS
printenv | grep MPI
echo "Environment above, where is P2P and MPI1 "
which IMB-P2P
which IMB-MPI1 
fi_info -e 
echo "HPL Test" 
date
# mpirun  -iface ib0 -verbose IMB-P2P
echo "P2P 2 Node"
# mpirun -l -v -n 2  -iface ib0  IMB-P2P 
mpirun -l -v -n 2   IMB-P2P 
mpirun -l -v -n 2   IMB-MPI1
#
# echo "XHPL Run"
date
mpirun  -n 40  /utils/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/benchmarks/mp_linpack/xhpl_intel64_dynamic 
date
# mpirun -l -v -n 4  -iface ib0  IMB-P2P 
# echo "MPI1 2 Node"
# mpirun -l -v -n 2  -iface ib0  IMB-MPI1
# echo "MPI1 4 Node"
# mpirun -l -v -n 4  -iface ib0  IMB-MPI1
# echo "AdventOfCode"
# cd ~/AdventOfCode
# source venv/bin/activate 
# python3 Day1.py
# python3 Day2.py 
# python3 Day3.py 
# python3 Day4.py
# python3 Day5.py
echo "End"