Segmentation fault with "flexible" mpi subroutine

[Wee_Beng_T_]

Hi,

I have an mpi subroutine which copies different variables into an integer and real array, send to different procs using MPI, and then updates the variables to their new values using the sent integer and real array.

It works and now because I have different cases, I use dynamic allocation instead. But when I use dynamic allocation, it doesn't work anymore, even with the original case. It gives segmentation fault when MPI_ALLGATHERV is called.

My new code is:

subroutine mpi_IIB_I_equal_to_global(IIB_equal_cell_no_global,I_equal_cell_no_global,IIB_equal_cells,I_equal_cells,IIB_global_cells,I_global_cells)

!to copy from equal to global IIB,I

!real and integers

!size dep on interpolation_choice

integer, intent(in) :: IIB_equal_cell_no_global(0:num_procs-1),I_equal_cell_no_global(0:num_procs-1)

integer :: ijk,ijk2,sta_no_int(0:num_procs-1),sta_no_real(0:num_procs-1),ierr,total_IIB_cell_no,plus1_int,plus1_real,IIB_equal_cell_no,total_I_cell_no,I_equal_cell_no,vec_chg_size,vec_chg_size2,vec_chg_size3,vec_chg_size4

!integer :: total_IIB_cell_no,total_I_cell_no,plus1_int,plus1_real

!integer :: tmp_equal_int(IIB_equal_cell_no_global(myid)*19 + I_equal_cell_no_global(myid)*6),tmp_global_int(sum(IIB_equal_cell_no_global)*19 + sum(I_equal_cell_no_global)*6)

integer, allocatable :: tmp_equal_int(:),tmp_global_int(:)

integer :: IIB_I_equal_cell_no_int_elements,IIB_I_equal_cell_no_real_elements,IIB_I_equal_cell_no_global_int_elements(0:num_procs-1),IIB_I_equal_cell_no_global_real_elements(0:num_procs-1),IIB_counter,I_counter

type(IIB_cell), intent(in) :: IIB_equal_cells(IIB_equal_cell_no_global(myid))

type(IIB_cell), intent(inout) :: IIB_global_cells(sum(IIB_equal_cell_no_global))

type(I_cell), intent(in) :: I_equal_cells(I_equal_cell_no_global(myid))

type(I_cell), intent(inout) :: I_global_cells(sum(I_equal_cell_no_global))

!real(8) :: tmp_equal_real(IIB_equal_cell_no_global(myid)*34 + I_equal_cell_no_global(myid)),tmp_global_real(sum(IIB_equal_cell_no_global)*34 + sum(I_equal_cell_no_global))

real(8), allocatable :: tmp_equal_real(:),tmp_global_real(:)

!depending on choice, size of array is different

if (interpolation_choice == 1) then

   vec_chg_size = 19;   vec_chg_size2 = 34; vec_chg_size3 = 18; vec_chg_size4 = 33
    
else if (interpolation_choice == 2) then

    vec_chg_size = 85;  vec_chg_size2 = 56; vec_chg_size3 = 84; vec_chg_size4 = 55
    
end if

!allocate suitable real and int array size

allocate (tmp_equal_int(IIB_equal_cell_no_global(myid)*vec_chg_size + I_equal_cell_no_global(myid)*6)); allocate (tmp_global_int(IIB_equal_cell_no_global(myid)*vec_chg_size + I_equal_cell_no_global(myid)*6))

allocate (tmp_equal_real(IIB_equal_cell_no_global(myid)*vec_chg_size2 + I_equal_cell_no_global(myid)),tmp_global_real(sum(IIB_equal_cell_no_global)*vec_chg_size2 + sum(I_equal_cell_no_global)))

total_IIB_cell_no = sum(IIB_equal_cell_no_global);  total_I_cell_no = sum(I_equal_cell_no_global)

IIB_equal_cell_no = IIB_equal_cell_no_global(myid); I_equal_cell_no = I_equal_cell_no_global(myid)

sta_no_int = 0

IIB_I_equal_cell_no_global_int_elements(0) = IIB_equal_cell_no_global(0)*vec_chg_size + I_equal_cell_no_global(0)*6

do ijk = 1,num_procs -1

    sta_no_int(ijk) = sum(IIB_equal_cell_no_global(0:ijk - 1))*vec_chg_size + sum(I_equal_cell_no_global(0:ijk - 1))*6
    
    IIB_I_equal_cell_no_global_int_elements(ijk) = IIB_equal_cell_no_global(ijk)*vec_chg_size + I_equal_cell_no_global(ijk)*6

end do

sta_no_real = 0

IIB_I_equal_cell_no_global_real_elements(0) = IIB_equal_cell_no_global(0)*vec_chg_size2 + I_equal_cell_no_global(0)

do ijk = 1,num_procs -1

    sta_no_real(ijk) = sum(IIB_equal_cell_no_global(0:ijk - 1))*vec_chg_size2 + sum(I_equal_cell_no_global(0:ijk - 1))
    
    IIB_I_equal_cell_no_global_real_elements(ijk) = IIB_equal_cell_no_global(ijk)*vec_chg_size2 + I_equal_cell_no_global(ijk)

end do

plus1_int = 1;  plus1_real = 1

do ijk = 1,IIB_equal_cell_no

!copy local variables to int array

    tmp_equal_int(plus1_int:plus1_int + 2) = IIB_equal_cells(ijk)%ijk
    
    !tmp_equal_int(plus1_int + 3:plus1_int + 11) = IIB_equal_cells(ijk)%int_template_ijk
    
    tmp_equal_int(plus1_int + 3) = IIB_equal_cells(ijk)%v_no
    
    tmp_equal_int(plus1_int + 4) = IIB_equal_cells(ijk)%s_no
    
    tmp_equal_int(plus1_int + 5) = IIB_equal_cells(ijk)%v_no2
    
    tmp_equal_int(plus1_int + 6) = IIB_equal_cells(ijk)%s_no2
    
    tmp_equal_int(plus1_int + 7) = IIB_equal_cells(ijk)%stencil_no
    
    tmp_equal_int(plus1_int + 8) = IIB_equal_cells(ijk)%e_no
    
    tmp_equal_int(plus1_int + 9) = IIB_equal_cells(ijk)%near_2_body
    
    tmp_equal_int(plus1_int + 10:plus1_int + vec_chg_size3) = IIB_equal_cells(ijk)%int_template_ijk

!copy local variables to real array
    
    tmp_equal_real(plus1_real:plus1_real + 2) = IIB_equal_cells(ijk)%xyz
    
    tmp_equal_real(plus1_real + 3:plus1_real + 5) = IIB_equal_cells(ijk)%IB_xyz
    
    tmp_equal_real(plus1_real + 6:plus1_real + 8) = IIB_equal_cells(ijk)%IB_xyz_old
    
    tmp_equal_real(plus1_real + 9:plus1_real + 11) = IIB_equal_cells(ijk)%IB_xyz2
    
    !tmp_equal_real(plus1_real + 12:plus1_real + 15) = IIB_equal_cells(ijk)%int_template_coef
    
    tmp_equal_real(plus1_real + 12) = IIB_equal_cells(ijk)%vel_b
    
    tmp_equal_real(plus1_real + 13) = IIB_equal_cells(ijk)%vel_b2
    
    tmp_equal_real(plus1_real + 14:plus1_real + 17) = IIB_equal_cells(ijk)%int_template_coef_matrix(1,:)
    
    tmp_equal_real(plus1_real + 18:plus1_real + 21) = IIB_equal_cells(ijk)%int_template_coef_matrix(2,:)
    
    tmp_equal_real(plus1_real + 22:plus1_real + 25) = IIB_equal_cells(ijk)%int_template_coef_matrix(3,:)
    
    tmp_equal_real(plus1_real + 26:plus1_real + 29) = IIB_equal_cells(ijk)%int_template_coef_matrix(4,:)
    
    tmp_equal_real(plus1_real + 30:plus1_real + vec_chg_size4) = IIB_equal_cells(ijk)%int_template_coef
        
    plus1_int = plus1_int + vec_chg_size;     plus1_real = plus1_real + vec_chg_size2
    
end do

do ijk = 1,I_equal_cell_no

    tmp_equal_int(plus1_int:plus1_int + 2) = I_equal_cells(ijk)%ijk
    
    tmp_equal_int(plus1_int + 3) = I_equal_cells(ijk)%v_no
    
    tmp_equal_int(plus1_int + 4) = I_equal_cells(ijk)%s_no
    
    tmp_equal_int(plus1_int + 5) = I_equal_cells(ijk)%e_no
    
    tmp_equal_real(plus1_real) = I_equal_cells(ijk)%vel
        
    plus1_int = plus1_int + 6;     plus1_real = plus1_real + 1
    
end do

IIB_I_equal_cell_no_int_elements = IIB_equal_cell_no*vec_chg_size + I_equal_cell_no*6

IIB_I_equal_cell_no_real_elements = IIB_equal_cell_no*vec_chg_size2 + I_equal_cell_no

!mpi communication

CALL MPI_ALLGATHERV(tmp_equal_int,IIB_I_equal_cell_no_int_elements,MPI_INTEGER,tmp_global_int,IIB_I_equal_cell_no_global_int_elements,sta_no_int,MPI_INTEGER,MPI_COMM_WORLD,ierr)

CALL MPI_ALLGATHERV(tmp_equal_real,IIB_I_equal_cell_no_real_elements,MPI_DOUBLE_PRECISION,tmp_global_real,IIB_I_equal_cell_no_global_real_elements,sta_no_real,MPI_DOUBLE_PRECISION,MPI_COMM_WORLD,ierr)

plus1_int = 1;  plus1_real = 1;  IIB_counter = 1;  I_counter = 1

do ijk = 0,num_procs - 1

    do ijk2 = 1,IIB_equal_cell_no_global(ijk)

!copy array back to local variables

        IIB_global_cells(IIB_counter)%ijk = tmp_global_int(plus1_int:plus1_int + 2)
        
        !IIB_global_cells(IIB_counter)%int_template_ijk = tmp_global_int(plus1_int + 3:plus1_int + 11)
        
        IIB_global_cells(IIB_counter)%v_no = tmp_global_int(plus1_int + 3)
        
        IIB_global_cells(IIB_counter)%s_no = tmp_global_int(plus1_int + 4)
        
        IIB_global_cells(IIB_counter)%v_no2 = tmp_global_int(plus1_int + 5)
        
        IIB_global_cells(IIB_counter)%s_no2 = tmp_global_int(plus1_int + 6)
        
        IIB_global_cells(IIB_counter)%stencil_no = tmp_global_int(plus1_int + 7)
        
        IIB_global_cells(IIB_counter)%e_no = tmp_global_int(plus1_int + 8)
        
        IIB_global_cells(IIB_counter)%near_2_body = tmp_global_int(plus1_int + 9)
        
        IIB_global_cells(IIB_counter)%int_template_ijk = tmp_global_int(plus1_int + 10:plus1_int + vec_chg_size3)
        
        IIB_global_cells(IIB_counter)%xyz = tmp_global_real(plus1_real:plus1_real + 2)
        
        IIB_global_cells(IIB_counter)%IB_xyz = tmp_global_real(plus1_real + 3:plus1_real + 5)
    
        IIB_global_cells(IIB_counter)%IB_xyz_old = tmp_global_real(plus1_real + 6:plus1_real + 8)
        
        IIB_global_cells(IIB_counter)%IB_xyz2 = tmp_global_real(plus1_real + 9:plus1_real + 11)
        
        !IIB_global_cells(IIB_counter)%int_template_coef = tmp_global_real(plus1_real + 12:plus1_real + 15)
        
        IIB_global_cells(IIB_counter)%vel_b = tmp_global_real(plus1_real + 12)
        
        IIB_global_cells(IIB_counter)%vel_b2 = tmp_global_real(plus1_real + 13)
        
        IIB_global_cells(IIB_counter)%int_template_coef_matrix(1,:) = tmp_global_real(plus1_real + 14:plus1_real + 17)
        
        IIB_global_cells(IIB_counter)%int_template_coef_matrix(2,:) = tmp_global_real(plus1_real + 18:plus1_real + 21)
        
        IIB_global_cells(IIB_counter)%int_template_coef_matrix(3,:) = tmp_global_real(plus1_real + 22:plus1_real + 25)
        
        IIB_global_cells(IIB_counter)%int_template_coef_matrix(4,:) = tmp_global_real(plus1_real + 26:plus1_real + 29)
        
        IIB_global_cells(IIB_counter)%int_template_coef = tmp_global_real(plus1_real + 30:plus1_real + vec_chg_size4)
            
        plus1_int = plus1_int + vec_chg_size; plus1_real = plus1_real + vec_chg_size2; IIB_counter = IIB_counter + 1
        
    end do

    do ijk2 = 1,I_equal_cell_no_global(ijk)

        I_global_cells(I_counter)%ijk = tmp_global_int(plus1_int:plus1_int + 2)
        
        I_global_cells(I_counter)%v_no = tmp_global_int(plus1_int + 3)
    
        I_global_cells(I_counter)%s_no = tmp_global_int(plus1_int + 4)
    
        I_global_cells(I_counter)%e_no = tmp_global_int(plus1_int + 5)
        
        I_global_cells(I_counter)%vel = tmp_global_real(plus1_real)
            
        plus1_int = plus1_int + 6;     plus1_real = plus1_real + 1; I_counter = I_counter + 1
        
    end do
    
end do

!call MPI_Barrier(MPI_COMM_WORLD,ierr)

deallocate (tmp_equal_int); deallocate(tmp_global_int)

deallocate (tmp_equal_real); deallocate(tmp_global_real)

end subroutine mpi_IIB_I_equal_to_global

[James_T_Intel]

Look at how you are allocating tmp_global_int.  When you were using static allocation, you used

tmp_global_int(sum(IIB_equal_cell_no_global)*19 + sum(I_equal_cell_no_global)*6)

With dynamic allocation, you are using

tmp_global_int(IIB_equal_cell_no_global(myid)*vec_chg_size + I_equal_cell_no_global(myid)*6)

In your first MPI_Allgatherv, you are collecting into tmp_global_int.  Your dynamic allocation doesn't appear to have enough space for everything you are gathering.

James.

[Wee_Beng_T_]

Oh, that's really careless of me.

Thank you so much for pointing it out.