Unable to generate trace file (*.stf)

[Lim__Hyun]

Hello all,

I tried to generate trace file for performance profiling the code. I am testing in stampede2 with the module lists

Currently Loaded Modules:
  1) git/2.9.0       3) xalt/1.7.7   5) intel/17.0.4   7) python/2.7.13   9) hdf5/1.8.16  11) petsc/3.7
  2) autotools/1.1   4) TACC         6) impi/17.0.3    8) gsl/2.3        10) papi/5.5.1   12) itac/17.0.3

ITAC help shows how I can generate the trace file from certain simulation. Below is the configuration result from the code

# Whenever this version string changes, the application is configured
# and rebuilt from scratch
VERSION = stampede2-2017-10-03

CPP = cpp
FPP = cpp
CC  = mpicc
CXX = mpicxx
F77 = ifort
F90 = ifort

CPPFLAGS = -g -trace -D_XOPEN_SOURCE -D_XOPEN_SOURCE_EXTENDED
FPPFLAGS = -g -trace -traditional
CFLAGS   = -g -trace -traceback -debug all -xCORE-AVX2 -axMIC-AVX512 -align -std=gnu99
CXXFLAGS = -g -trace -traceback -debug all -xCORE-AVX2 -axMIC-AVX512 -align -std=gnu++11
F77FLAGS = -g -trace -traceback -debug all -xCORE-AVX2 -axMIC-AVX512 -align -pad -safe-cray-ptr
F90FLAGS = -g -trace -traceback -debug all -xCORE-AVX2 -axMIC-AVX512 -align -pad -safe-cray-ptr

LDFLAGS = -rdynamic -xCORE-AVX2 -axMIC-AVX512

C_LINE_DIRECTIVES = yes
F_LINE_DIRECTIVES = yes

VECTORISE                = yes
VECTORISE_ALIGNED_ARRAYS = no
VECTORISE_INLINE         = no

DEBUG = no
CPP_DEBUG_FLAGS = -DCARPET_DEBUG
FPP_DEBUG_FLAGS = -DCARPET_DEBUG
C_DEBUG_FLAGS   = -O0
CXX_DEBUG_FLAGS = -O0
F77_DEBUG_FLAGS = -O0 -check bounds -check format
F90_DEBUG_FLAGS = -O0 -check bounds -check format

OPTIMISE = yes
CPP_OPTIMISE_FLAGS = # -DCARPET_OPTIMISE -DNDEBUG
FPP_OPTIMISE_FLAGS = # -DCARPET_OPTIMISE -DNDEBUG
C_OPTIMISE_FLAGS   = -Ofast
CXX_OPTIMISE_FLAGS = -Ofast
F77_OPTIMISE_FLAGS = -Ofast
F90_OPTIMISE_FLAGS = -Ofast

CPP_NO_OPTIMISE_FLAGS  =
FPP_NO_OPTIMISE_FLAGS  =
C_NO_OPTIMISE_FLAGS    = -O0
CXX_NO_OPTIMISE_FLAGS  = -O0
CUCC_NO_OPTIMISE_FLAGS =
F77_NO_OPTIMISE_FLAGS  = -O0
F90_NO_OPTIMISE_FLAGS  = -O0

PROFILE = no
CPP_PROFILE_FLAGS =
FPP_PROFILE_FLAGS =
C_PROFILE_FLAGS   = -pg
CXX_PROFILE_FLAGS = -pg
F77_PROFILE_FLAGS = -pg
F90_PROFILE_FLAGS = -pg

OPENMP           = yes
CPP_OPENMP_FLAGS = -fopenmp
FPP_OPENMP_FLAGS = -fopenmp
C_OPENMP_FLAGS   = -fopenmp
CXX_OPENMP_FLAGS = -fopenmp
F77_OPENMP_FLAGS = -fopenmp
F90_OPENMP_FLAGS = -fopenmp

WARN           = yes
CPP_WARN_FLAGS =
FPP_WARN_FLAGS =
C_WARN_FLAGS   =
CXX_WARN_FLAGS =
F77_WARN_FLAGS =
F90_WARN_FLAGS =

BLAS_DIR  = NO_BUILD
BLAS_LIBS = -mkl

HWLOC_DIR        = NO_BUILD
HWLOC_EXTRA_LIBS = numa

LAPACK_DIR  = NO_BUILD
LAPACK_LIBS = -mkl

OPENBLAS_DIR  = NO_BUILD
OPENBLAS_LIBS = -mkl

HDF5_DIR = /opt/apps/intel17/hdf5/1.8.16/x86_64

BOOST_DIR = /opt/apps/intel17/boost/1.64

GSL_DIR = /opt/apps/intel17/gsl/2.3

FFTW3_DIR = NO_BUILD
FFTW3_INC_DIRS = /opt/intel/compilers_and_libraries_2017.4.196/linux/mkl/include/fftw
FFTW3_LIBS = -mkl

PAPI_DIR = /opt/apps/papi/5.5.1

PETSC_DIR = /home1/apps/intel17/impi17_0/petsc/3.7/knightslanding
PETSC_LAPACK_EXTRA_LIBS = -mkl

PTHREADS_DIR = NO_BUILD

I am using mpicc/mpicxx with -trace flag to enable to use ITAC. Then, I use bellow to submit the job. First, runscript is generated

#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

module unload mvapich2
module load impi/17.0.3
module list

echo "Checking:"
pwd
hostname
date

echo "Environment:"
#export I_MPI_FABRICS=shm:ofa
#export I_MPI_MIC=1
#export I_MPI_OFA_ADAPTER_NAME=mlx4_0
export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
export CACTUS_SET_THREAD_BINDINGS=1
export CXX_MAX_TASKS=500
export GMON_OUT_PREFIX=gmon.out
export OMP_MAX_TASKS=500
export OMP_NUM_THREADS=@NUM_THREADS@
export OMP_STACKSIZE=8192       # kByte
export PTHREAD_MAX_TASKS=500
env | sort > SIMFACTORY/ENVIRONMENT
echo ${SLURM_NODELIST} > NODES

echo "Starting:"
export CACTUS_STARTTIME=$(date +%s)
export VT_PCTRACE=1
time ibrun -trace @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."

As you can see above, I use iburn -trace. Then below is the submit script

#! /bin/bash
#SBATCH -A @ALLOCATION@
#SBATCH -p @QUEUE@
#SBATCH -t @WALLTIME@
#SBATCH -N @NODES@ -n @NUM_PROCS@
#SBATCH @("@CHAINED_JOB_ID@" != "" ? "-d afterany:@CHAINED_JOB_ID@" : "")@
#SBATCH -J @SHORT_SIMULATION_NAME@
#SBATCH --mail-type=ALL
#SBATCH --mail-user=@EMAIL@
#SBATCH -o @RUNDIR@/@SIMULATION_NAME@.out
#SBATCH -e @RUNDIR@/@SIMULATION_NAME@.err
cd @SOURCEDIR@
@SIMFACTORY@ run @SIMULATION_NAME@ --machine=@MACHINE@ --restart-id=@RESTART_ID@ @FROM_RESTART_COMMAND@

I guess that's what I need to generate trace file. I sent several simple jobs to check it but I didn't get any file after simulation. Simulation was completed without the problem so I am stuck now.

Does anyone have idea about this? The code I would like to check is called einstein-toolkit