$ mpirun -genvall -genv I_MPI_DEBUG 5 -genv I_MPI_HYDRA_DEBUG 1 -genv I_MPI_FABRICS=shm:dapl -machinefile ./nodes -n 96 ./incompact3d > log

[60] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[40] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[44] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[68] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[72] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[80] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[36] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[4] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[8] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[64] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[84] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[48] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[76] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[92] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[0] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[24] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[32] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[88] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[12] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[16] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[20] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[52] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[28] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[56] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[33] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[21] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[9] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[37] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[5] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[73] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[1] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[25] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[77] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[13] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[41] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[49] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[61] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[65] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[89] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[29] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[45] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[53] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[69] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[81] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[85] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[57] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[93] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[17] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[2] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[10] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[90] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[6] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[50] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[74] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[54] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[62] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[78] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[70] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[82] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[18] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[22] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[58] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[38] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[66] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[86] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[26] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[30] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[34] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[42] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[14] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[46] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[94] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[3] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[7] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[11] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[15] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[19] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[27] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[31] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[35] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[39] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[43] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[47] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[51] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[55] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[59] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[63] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[67] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[75] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[79] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[83] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[87] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[91] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[95] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[23] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[71] DAPL startup(): trying to open default DAPL provider from dat registry: ofa-v2-mlx4_0-1
[36] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[36] MPI startup(): shm and dapl data transfer modes
[68] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[68] MPI startup(): shm and dapl data transfer modes
[76] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[48] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[76] MPI startup(): shm and dapl data transfer modes
[48] MPI startup(): shm and dapl data transfer modes
[56] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[12] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[56] MPI startup(): shm and dapl data transfer modes
[12] MPI startup(): shm and dapl data transfer modes
[0] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[0] MPI startup(): shm and dapl data transfer modes
[52] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[80] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[52] MPI startup(): shm and dapl data transfer modes
[80] MPI startup(): shm and dapl data transfer modes
[40] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[40] MPI startup(): shm and dapl data transfer modes
[32] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[32] MPI startup(): shm and dapl data transfer modes
[72] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[72] MPI startup(): shm and dapl data transfer modes
[8] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[8] MPI startup(): shm and dapl data transfer modes
[16] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[16] MPI startup(): shm and dapl data transfer modes
[84] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[64] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[44] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[64] MPI startup(): shm and dapl data transfer modes
[44] MPI startup(): shm and dapl data transfer modes
[84] MPI startup(): shm and dapl data transfer modes
[92] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[33] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[60] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[33] MPI startup(): shm and dapl data transfer modes
[60] MPI startup(): shm and dapl data transfer modes
[4] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[4] MPI startup(): shm and dapl data transfer modes
[24] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[24] MPI startup(): shm and dapl data transfer modes
[88] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[20] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[5] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[88] MPI startup(): shm and dapl data transfer modes
[20] MPI startup(): shm and dapl data transfer modes
[5] MPI startup(): shm and dapl data transfer modes
[9] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[9] MPI startup(): shm and dapl data transfer modes
[85] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[85] MPI startup(): shm and dapl data transfer modes
[73] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[69] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[73] MPI startup(): shm and dapl data transfer modes
[69] MPI startup(): shm and dapl data transfer modes
[21] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[21] MPI startup(): shm and dapl data transfer modes
[77] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[28] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[28] MPI startup(): shm and dapl data transfer modes
[77] MPI startup(): shm and dapl data transfer modes
[93] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[93] MPI startup(): shm and dapl data transfer modes
[25] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[89] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[89] MPI startup(): shm and dapl data transfer modes
[25] MPI startup(): shm and dapl data transfer modes
[29] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[29] MPI startup(): shm and dapl data transfer modes
[17] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[92] MPI startup(): shm and dapl data transfer modes
[1] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[17] MPI startup(): shm and dapl data transfer modes
[1] MPI startup(): shm and dapl data transfer modes
[57] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[57] MPI startup(): shm and dapl data transfer modes
[41] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[63] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[63] MPI startup(): shm and dapl data transfer modes
[41] MPI startup(): shm and dapl data transfer modes
[61] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[61] MPI startup(): shm and dapl data transfer modes
[13] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[78] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[78] MPI startup(): shm and dapl data transfer modes
[2] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[2] MPI startup(): shm and dapl data transfer modes
[11] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[11] MPI startup(): shm and dapl data transfer modes
[59] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[50] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[13] MPI startup(): shm and dapl data transfer modes
[59] MPI startup(): shm and dapl data transfer modes
[37] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[37] MPI startup(): shm and dapl data transfer modes
[50] MPI startup(): shm and dapl data transfer modes
[49] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[49] MPI startup(): shm and dapl data transfer modes
[53] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[53] MPI startup(): shm and dapl data transfer modes
[65] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[65] MPI startup(): shm and dapl data transfer modes
[81] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[81] MPI startup(): shm and dapl data transfer modes
[45] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[47] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[45] MPI startup(): shm and dapl data transfer modes
[47] MPI startup(): shm and dapl data transfer modes
[43] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[43] MPI startup(): shm and dapl data transfer modes
[90] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[90] MPI startup(): shm and dapl data transfer modes
[10] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[10] MPI startup(): shm and dapl data transfer modes
[51] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[51] MPI startup(): shm and dapl data transfer modes
[70] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[14] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[70] MPI startup(): shm and dapl data transfer modes
[82] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[14] MPI startup(): shm and dapl data transfer modes
[82] MPI startup(): shm and dapl data transfer modes
[7] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[7] MPI startup(): shm and dapl data transfer modes
[35] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[35] MPI startup(): shm and dapl data transfer modes
[55] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[55] MPI startup(): shm and dapl data transfer modes
[71] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[71] MPI startup(): shm and dapl data transfer modes
[15] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[19] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[15] MPI startup(): shm and dapl data transfer modes
[19] MPI startup(): shm and dapl data transfer modes
[83] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[6] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[83] MPI startup(): shm and dapl data transfer modes
[46] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[54] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[67] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[46] MPI startup(): shm and dapl data transfer modes
[54] MPI startup(): shm and dapl data transfer modes
[67] MPI startup(): shm and dapl data transfer modes
[3] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[42] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[3] MPI startup(): shm and dapl data transfer modes
[34] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[75] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[42] MPI startup(): shm and dapl data transfer modes
[34] MPI startup(): shm and dapl data transfer modes
[75] MPI startup(): shm and dapl data transfer modes
[31] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[31] MPI startup(): shm and dapl data transfer modes
[30] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[30] MPI startup(): shm and dapl data transfer modes
[66] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[66] MPI startup(): shm and dapl data transfer modes
[23] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[87] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[87] MPI startup(): shm and dapl data transfer modes
[23] MPI startup(): shm and dapl data transfer modes
[86] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[86] MPI startup(): shm and dapl data transfer modes
[27] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[94] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[94] MPI startup(): shm and dapl data transfer modes
[91] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[18] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[27] MPI startup(): shm and dapl data transfer modes
[91] MPI startup(): shm and dapl data transfer modes
[18] MPI startup(): shm and dapl data transfer modes
[95] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[79] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[58] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[95] MPI startup(): shm and dapl data transfer modes
[79] MPI startup(): shm and dapl data transfer modes
[58] MPI startup(): shm and dapl data transfer modes
[62] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[62] MPI startup(): shm and dapl data transfer modes
[22] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[22] MPI startup(): shm and dapl data transfer modes
[26] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[26] MPI startup(): shm and dapl data transfer modes
[38] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[38] MPI startup(): shm and dapl data transfer modes
[39] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[39] MPI startup(): shm and dapl data transfer modes
[6] MPI startup(): shm and dapl data transfer modes
[74] MPI startup(): DAPL provider ofa-v2-mlx4_0-1
[74] MPI startup(): shm and dapl data transfer modes
[0] MPI startup(): Rank    Pid      Node name   Pin cpu

[0] MPI startup(): 0       43426    cerrado01n  0

[0] MPI startup(): 1       35505    cerrado04n  0

[0] MPI startup(): 2       35478    cerrado03n  0

[0] MPI startup(): 3       34829    cerrado02n  0

[0] MPI startup(): 4       43427    cerrado01n  12

[0] MPI startup(): 5       35507    cerrado04n  12

[0] MPI startup(): 6       35479    cerrado03n  12

[0] MPI startup(): 7       34831    cerrado02n  12

[0] MPI startup(): 8       43428    cerrado01n  2

[0] MPI startup(): 9       35508    cerrado04n  2

[0] MPI startup(): 10      35481    cerrado03n  2

[0] MPI startup(): 11      34832    cerrado02n  2

[0] MPI startup(): 12      43429    cerrado01n  14

[0] MPI startup(): 13      35509    cerrado04n  14

[0] MPI startup(): 14      35482    cerrado03n  14

[0] MPI startup(): 15      34833    cerrado02n  14

[0] MPI startup(): 16      43430    cerrado01n  4

[0] MPI startup(): 17      35510    cerrado04n  4

[0] MPI startup(): 18      35483    cerrado03n  4

[0] MPI startup(): 19      34834    cerrado02n  4

[0] MPI startup(): 20      43431    cerrado01n  16

[0] MPI startup(): 21      35511    cerrado04n  16

[0] MPI startup(): 22      35484    cerrado03n  16

[0] MPI startup(): 23      34835    cerrado02n  16

[0] MPI startup(): 24      43432    cerrado01n  6

[0] MPI startup(): 25      35512    cerrado04n  6

[0] MPI startup(): 26      35485    cerrado03n  6

[0] MPI startup(): 27      34836    cerrado02n  6

[0] MPI startup(): 28      43433    cerrado01n  18

[0] MPI startup(): 29      35513    cerrado04n  18

[0] MPI startup(): 30      35486    cerrado03n  18

[0] MPI startup(): 31      34837    cerrado02n  18

[0] MPI startup(): 32      43435    cerrado01n  8

[0] MPI startup(): 33      35514    cerrado04n  8

[0] MPI startup(): 34      35487    cerrado03n  8

[0] MPI startup(): 35      34838    cerrado02n  8

[0] MPI startup(): 36      43436    cerrado01n  20

[0] MPI startup(): 37      35515    cerrado04n  20

[0] MPI startup(): 38      35488    cerrado03n  20

[0] MPI startup(): 39      34839    cerrado02n  20

[0] MPI startup(): 40      43437    cerrado01n  10

[0] MPI startup(): 41      35516    cerrado04n  10

[0] MPI startup(): 42      35489    cerrado03n  10

[0] MPI startup(): 43      34840    cerrado02n  10

[0] MPI startup(): 44      43438    cerrado01n  22

[0] MPI startup(): 45      35517    cerrado04n  22

[0] MPI startup(): 46      35490    cerrado03n  22

[0] MPI startup(): 47      34841    cerrado02n  22

[0] MPI startup(): 48      43439    cerrado01n  1

[0] MPI startup(): 49      35518    cerrado04n  1

[0] MPI startup(): 50      35491    cerrado03n  1

[0] MPI startup(): 51      34842    cerrado02n  1

[0] MPI startup(): 52      43440    cerrado01n  13

[0] MPI startup(): 53      35519    cerrado04n  13

[0] MPI startup(): 54      35492    cerrado03n  13

[0] MPI startup(): 55      34843    cerrado02n  13

[0] MPI startup(): 56      43441    cerrado01n  3

[0] MPI startup(): 57      35520    cerrado04n  3

[0] MPI startup(): 58      35493    cerrado03n  3

[0] MPI startup(): 59      34844    cerrado02n  3

[0] MPI startup(): 60      43442    cerrado01n  15

[0] MPI startup(): 61      35521    cerrado04n  15

[0] MPI startup(): 62      35494    cerrado03n  15

[0] MPI startup(): 63      34845    cerrado02n  15

[0] MPI startup(): 64      43443    cerrado01n  5

[0] MPI startup(): 65      35522    cerrado04n  5

[0] MPI startup(): 66      35495    cerrado03n  5

[0] MPI startup(): 67      34846    cerrado02n  5

[0] MPI startup(): 68      43444    cerrado01n  17

[0] MPI startup(): 69      35523    cerrado04n  17

[0] MPI startup(): 70      35496    cerrado03n  17

[0] MPI startup(): 71      34847    cerrado02n  17

[0] MPI startup(): 72      43445    cerrado01n  7

[0] MPI startup(): 73      35524    cerrado04n  7

[0] MPI startup(): 74      35497    cerrado03n  7

[0] MPI startup(): 75      34848    cerrado02n  7

[0] MPI startup(): 76      43446    cerrado01n  19

[0] MPI startup(): 77      35525    cerrado04n  19

[0] MPI startup(): 78      35498    cerrado03n  19

[0] MPI startup(): 79      34849    cerrado02n  19

[0] MPI startup(): 80      43447    cerrado01n  9

[0] MPI startup(): 81      35526    cerrado04n  9

[0] MPI startup(): 82      35499    cerrado03n  9

[0] MPI startup(): 83      34850    cerrado02n  9

[0] MPI startup(): 84      43448    cerrado01n  21

[0] MPI startup(): 85      35527    cerrado04n  21

[0] MPI startup(): 86      35500    cerrado03n  21

[0] MPI startup(): 87      34851    cerrado02n  21

[0] MPI startup(): 88      43449    cerrado01n  11

[0] MPI startup(): 89      35528    cerrado04n  11

[0] MPI startup(): 90      35501    cerrado03n  11

[0] MPI startup(): 91      34852    cerrado02n  11

[0] MPI startup(): 92      43450    cerrado01n  23

[0] MPI startup(): 93      35529    cerrado04n  23

[0] MPI startup(): 94      35502    cerrado03n  23

[0] MPI startup(): 95      34853    cerrado02n  23

[0] MPI startup(): I_MPI_DEBUG=5
[0] MPI startup(): I_MPI_FABRICS=shm:dapl
[0] MPI startup(): I_MPI_PIN_MAPPING=24:0 0,4 12,8 2,12 14,16 4,20 16,24 6,28 18,32 8,36 20,40 10,44 22,48 1,52 13,56 3,60 15,64 5,68 17,72 7,76 19,80 9,84 21,88 11,92 23
 ===========================================================
 ======================Incompact3d==========================
 ===Copyright (c) 2012 Eric Lamballais and Sylvain Laizet===
 ===Modified by Ricardo Frantz==============================
 ===========================================================
 
 (lx,ly,lz)=   16.00000       2.000000       2.000000    
 (nx,ny,nz)=        1945         241         240
 (nx*ny*nz)=   112498800
 (p_row,p_col)=           8          12
 
 Open channel configuration
 Boundary condition : (nclx,ncly,nclz)=(2,2,0)
 High and low speed : u1=  2.00 and u2=  1.00
 Reynolds number Re :   5000.00000000
 Time step dt       :      0.00050000
 Passive scalar     : on
 Schmidt number     :   1.00
 Angle              :   0.00
 Immersed boundary  : off
 This simulation requieres   472.4949     GB for 3D fields
 This simulation requieres  0.9843665     GB for 2D fields
 Starting paraview subroutine...
 Writing 3d.xdmf!
 Writing 2d.xdmf!
 Writing maps.xdmf!
 Writing mean.xdmf!
 Initializing variables...
 [88:cerrado01n] unexpected disconnect completion event from [87:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 88
[16:cerrado01n] unexpected disconnect completion event from [15:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 16
[40:cerrado01n] unexpected disconnect completion event from [41:cerrado04n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 40
[92:cerrado01n] unexpected disconnect completion event from [57:cerrado04n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 92
[64:cerrado01n] unexpected disconnect completion event from [65:cerrado04n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 64
[80:cerrado01n] unexpected disconnect completion event from [79:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 80
[28:cerrado01n] unexpected disconnect completion event from [27:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 28
[76:cerrado01n] unexpected disconnect completion event from [75:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 76
[8:cerrado01n] unexpected disconnect completion event from [7:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 8
[32:cerrado01n] unexpected disconnect completion event from [34:cerrado03n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 32
[68:cerrado01n] unexpected disconnect completion event from [67:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 68
[20:cerrado01n] unexpected disconnect completion event from [19:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 20
[44:cerrado01n] unexpected disconnect completion event from [43:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 44
[56:cerrado01n] unexpected disconnect completion event from [57:cerrado04n]
[52:cerrado01n] unexpected disconnect completion event from [51:cerrado02n]
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 52
Assertion failed in file ../../dapl_conn_rc.c at line 1128: 0
internal ABORT - process 56
APPLICATION TERMINATED WITH THE EXIT STRING: Segmentation fault (signal 11)
