Thread 5 (Thread 0x7fe1906ec700 (LWP 18605)): #0 0x000000307b20b5e5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fe19181c92e in MPIDI_CH3I_Progress () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #2 0x00007fe19190a3f9 in MPIC_Wait () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #3 0x00007fe19190a153 in MPIC_Sendrecv () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #4 0x00007fe191903a93 in MPIC_Sendrecv_ft () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #5 0x00007fe1917f21c7 in MPIR_Allreduce_intra () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #6 0x00007fe1917f59ed in MPIR_Allreduce_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #7 0x00007fe19184a526 in MPIR_Get_contextid_sparse () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #8 0x00007fe19184a3a8 in MPIR_Get_contextid () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #9 0x00007fe191849fc8 in MPIR_Comm_copy () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #10 0x00007fe191842360 in MPIR_Comm_dup_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #11 0x00007fe191842054 in PMPI_Comm_dup () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #12 0x0000000000400eda in thread_fcn () #13 0x000000307b207d15 in start_thread () from /lib64/libpthread.so.0 #14 0x000000307a6f253d in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7fe18feeb700 (LWP 18608)): #0 0x000000307b20b5e5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fe19181c92e in MPIDI_CH3I_Progress () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #2 0x00007fe19190a3f9 in MPIC_Wait () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #3 0x00007fe19190a153 in MPIC_Sendrecv () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #4 0x00007fe191903a93 in MPIC_Sendrecv_ft () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #5 0x00007fe1917f21c7 in MPIR_Allreduce_intra () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #6 0x00007fe1917f59ed in MPIR_Allreduce_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #7 0x00007fe19184a526 in MPIR_Get_contextid_sparse () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #8 0x00007fe19184a3a8 in MPIR_Get_contextid () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #9 0x00007fe191849fc8 in MPIR_Comm_copy () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #10 0x00007fe191842360 in MPIR_Comm_dup_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #11 0x00007fe191842054 in PMPI_Comm_dup () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #12 0x0000000000400e66 in thread_fcn () #13 0x000000307b207d15 in start_thread () from /lib64/libpthread.so.0 #14 0x000000307a6f253d in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7fe18f6ea700 (LWP 18611)): #0 0x000000307b20de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x000000307b209ca6 in _L_lock_836 () from /lib64/libpthread.so.0 #2 0x000000307b209ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fe19181ae19 in MPIDI_CH3I_Progress () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #4 0x00007fe19190a3f9 in MPIC_Wait () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #5 0x00007fe19190a153 in MPIC_Sendrecv () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #6 0x00007fe191903a93 in MPIC_Sendrecv_ft () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #7 0x00007fe1917f21c7 in MPIR_Allreduce_intra () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #8 0x00007fe1917f59ed in MPIR_Allreduce_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #9 0x00007fe19184a526 in MPIR_Get_contextid_sparse () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #10 0x00007fe19184a3a8 in MPIR_Get_contextid () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #11 0x00007fe191849fc8 in MPIR_Comm_copy () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #12 0x00007fe191842360 in MPIR_Comm_dup_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #13 0x00007fe191842054 in PMPI_Comm_dup () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #14 0x0000000000400e66 in thread_fcn () #15 0x000000307b207d15 in start_thread () from /lib64/libpthread.so.0 #16 0x000000307a6f253d in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7fe18eee9700 (LWP 18612)): #0 0x000000307b20b5e5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fe19181c92e in MPIDI_CH3I_Progress () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #2 0x00007fe19190a3f9 in MPIC_Wait () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #3 0x00007fe19190a153 in MPIC_Sendrecv () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #4 0x00007fe191903a93 in MPIC_Sendrecv_ft () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #5 0x00007fe1917f21c7 in MPIR_Allreduce_intra () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #6 0x00007fe1917f59ed in MPIR_Allreduce_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #7 0x00007fe19184a526 in MPIR_Get_contextid_sparse () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #8 0x00007fe19184a3a8 in MPIR_Get_contextid () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #9 0x00007fe191849fc8 in MPIR_Comm_copy () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #10 0x00007fe191842360 in MPIR_Comm_dup_impl () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #11 0x00007fe191842054 in PMPI_Comm_dup () from /opt/soft1/intel-mpi/impi/4.1.3.049/intel64/lib/libmpi_mt.so.4 #12 0x0000000000400e66 in thread_fcn () #13 0x000000307b207d15 in start_thread () from /lib64/libpthread.so.0 #14 0x000000307a6f253d in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7fe19170eb40 (LWP 18604)): #0 0x000000307b208e60 in pthread_join () from /lib64/libpthread.so.0 #1 0x0000000000400ca8 in main ()