Index intrinsic function IVF return 0 for very big value

roy437 · ‎06-27-2022

Hi,

I have a program that turns a very large stl file (4.11GB) into a pts file "stl_to_pts.f90". An example stl file:

solid ASCII STL file generated with VxScan by Creaform.
facet normal 0.284589 0.628278 -0.724069
outer loop
vertex -1166.990479 -277.309601 656.863098
vertex -1166.887573 -276.861084 657.292725
vertex -1166.407104 -277.144348 657.235779
endloop
endfacet
facet normal 0.287695 0.632549 -0.719106
outer loop
vertex -1166.887573 -276.861084 657.292725
vertex -1166.523071 -276.255005 657.971680
vertex -1166.407104 -277.144348 657.235779
endloop
endfacet
facet normal 0.408622 0.642011 -0.648730
outer loop
vertex -1166.527466 -276.258850 657.991821
vertex -1166.468018 -276.235260 658.052612
vertex -1166.257812 -276.307129 658.113892
endloop
endfacet
facet normal 0.082224 0.975477 0.204167
outer loop
vertex -1166.523071 -276.255005 657.971680
vertex -1166.527466 -276.258850 657.991821
vertex -1166.257812 -276.307129 658.113892
endloop
endfacet
facet normal 0.466552 0.599195 -0.650611
outer loop
vertex -1166.523071 -276.255005 657.971680
vertex -1166.257812 -276.307129 658.113892
vertex -1166.407104 -277.144348 657.235779
endloop
endfacet
facet normal 0.302361 0.775017 -0.554911
outer loop
vertex -1166.468018 -276.235260 658.052612
vertex -1166.377808 -276.043121 658.370117
vertex -1166.155640 -276.129272 658.370850
endloop
endfacet
facet normal 0.409470 0.665844 -0.623687
outer loop
vertex -1166.468018 -276.235260 658.052612
vertex -1166.155640 -276.129272 658.370850
vertex -1166.257812 -276.307129 658.113892
endloop
endfacet
facet normal 0.316780 0.659006 -0.682174
outer loop
vertex -1166.407104 -277.144348 657.235779
vertex -1166.257812 -276.307129 658.113892
vertex -1165.399780 -277.246002 657.605347
endloop
endfacet
endsolid

Source code:

    module read_fis
       character *260 fisi
       character(:), allocatable :: str
    end module read_fis

    subroutine read_file_to_string
    use ifport
    use read_fis
    implicit none
    integer *4 :: iunit, istat
    integer *8 :: file_size
    character(len=1) :: c
    open(newunit = iunit, file = fisi, status = 'OLD', &
            form = 'UNFORMATTED', access = 'STREAM', iostat = istat)
    if (istat==0) then
        inquire(file=fisi, size=file_size )
        if (file_size > 0) then
            allocate(character(len=file_size ) :: str)
            read(iunit, pos=1, iostat = istat) str
            if (istat==0) then
                !make sure it was all read by trying to read more:
                read(iunit, pos = file_size+1, iostat = istat) c
                if (.not. IS_IOSTAT_END(istat)) then
                    write(*,*) 'Error: file was not completely read: ', trim(fisi)
                end if
            else
                write(*,*) 'Error reading file: ', trim(fisi)
            end if
            close(iunit, iostat=istat)
        else
            write(*,*) 'Error getting file size: ',  trim(fisi)
        end if
    else
        write(*,*) 'Error opening file: ', trim(fisi)
    end if
    return
    end

!   Main program
!===================================================================================

    use read_fis

    implicit none
    character(:), allocatable :: a
    character *260 fise
    character  *10 ext
    character   *1 cr
    character   *2 crlf

    integer *8 i, j, m, i1, i2, k
    integer *4 np, kp
    integer *4 :: iunit, istat

    cr = char(13)
    crlf = cr//char(10)

    call GETARG(1, fisi)
    call GETARG(2, fise)

    kp = index(fise, '.', back = .true.) + 1
    ext = fise(kp:)

    call read_file_to_string ! str = fisi
    m = len(str) ! /2
    allocate(character(len=m) :: a)  !   a = fise

    open(newunit = iunit, file = fise, status = 'NEW', &
            form = 'UNFORMATTED', access = 'STREAM', iostat = istat)
    if(istat == 0) then
        continue
    else
        write(*,*) 'Error opening file: ', trim(fise)
        stop
    end if

    i = 1
    i1 = 1

    if(trim(ext) == 'pts' .or. trim(ext) == 'PTS') then
      np = 0
      do
         k = index(str(i:), 'vertex', kind = 8)
         if(k > 0) then
         	 np = np + 1
             i = i + k + int8(6)
             j = i + index(str(i:), cr, kind =  - int8(2)
             i2 = i1 + j - i + int8(4)
             a(i1:i2) = str(i:j)//' 9'//crlf
             i1 = i2 + int8(1)
         else
             write(99,'(i0,t11,i0)') k, i
             write(iunit, pos = 1, iostat = istat) a(:i2-2)
             exit
         end if
      end do
    else
      do
         k = index(str(i:), 'vertex')
         if(k > 0) then
             i = i + k + 6
             j = i + index(str(i:), cr)
             i2 = i1 + j - i
             a(i1:i2) = str(i:j)
             i1 = i2 + 1
         else
             write(iunit, pos = 1, iostat = istat) a(:i2-2)
             exit
         end if
      end do
    end if

    deallocate(str)
    close(iunit)
    deallocate(a)

    stop
    end

I checked the "str" variable and it stores the entire stl file.

The program stops when k = 0 when it no longer finds the word "vertex". The last index found by the program is i = 121,513,698 although the last index in the stl file is 4,416,480,763. The question is: why does the index function return 0 after i = 121,513,698.

In "Project properties" it is set to "Default Integer kind": 8 (/ integer-size: 64), and in "Configuration Manager": x64, Release

I mention that I have:
I7-3770 processor, 16GB RAM, WINDOWS 10 Home / 64bit,
Intel Parallel Studio XE 2020 Update 4 Windows x64.

Thanks.

andrew_4619 · ‎06-30-2022

I had a quick look at your program, and I didn't see the problem, it could be a compiler bug with >32bit character subscripts. If you think it is a bug make a minimal sample code that shows the problem that is easier all round.

There are some strange things in your code the text STL is a formatted file yet you open it as unformatted for example.

Functionally you could to what yo are doing in a much simpler way IMO. I made an ran a simple example below.

program txt_stl_2_pts
    implicit none(type, external)
    character(132) :: gin
    integer :: iun1, iun2, ip, istat
    character(*), parameter :: GA ='(A)'
    open( newunit = iun1, file ='winder_bed_type_LH.stl', status = 'old', iostat = istat)
    if( istat /= 0 ) goto 900
    open(  newunit = iun2, file ='output.txt', status = 'unknown', iostat = istat )
    if( istat /= 0 ) goto 900
    do 
        read( iun1, ga, iostat = istat ) gin
        if( istat /= 0 ) goto 900
        ip = index(gin,'vertex')
        if ( ip > 0 )  write( iun2, ga) trim( gin(ip+6:) )
    enddo
    900 continue
    close (iun1, iostat = istat )
    close (iun2, iostat = istat )
end program txt_stl_2_pts

u

roy437 · ‎07-24-2022

Please test : https://we.tl/t-PGVsUr5zzZ

Filesize = 4.11 GB

Tell me time to run. Time to run is big. Tell me performances of your computer and OS.

Thx, J. Roy

jimdempseyatthecove · ‎06-30-2022

FWIW line 87 looks invalid. This should have produced a compilation error.

Jim Dempsey

roy437 · ‎07-24-2022

The correct line 87 is: j = i + index(str(i:), cr, kind=8) - int8(2)

For test there is file : https://we.tl/t-PGVsUr5zzZ

J. Roy

JohnNichols · ‎07-24-2022

What are the command line arguments?

JohnNichols · ‎07-24-2022

The first problem is you have a test file so large that even VEDIT cannot open it to look at it. I have never seen a text file this large and I deal with huge FEM models. I am not aware of a better editor than vedit for very large files.

The file is created as text numbers, often you run into a problem that the format statement that created the number is incorrect for the number, which happens a lot. But I cannot read your file to look for errors in the text.

I suggest you write a simple program that reads and checks each line for correctness, this is a common thing for DXF files for industrial applications in some locations.

JohnNichols · ‎07-24-2022

! call GETARG(1, fisi)
    !call GETARG(2, fise)
    fisi = "Test.stl"
    fise = "Test.out"
    

    kp = index(fise, '.', back = .true.) + 1
    ext = fise(kp:)

    call read_file_to_string ! str = fisi
    m = len(str) ! /2
    allocate(character(len=m) :: a)  !   a = fise

    open(newunit = iunit, file = fise, status = 'UNKNOWN', &
            form = 'UNFORMATTED', access = 'STREAM', iostat = istat)

Now I can run it in VS 2019 - see unknown, old means you have to delete the file to run it.

roy437 · ‎07-24-2022

Ultraedit 64-bit read very large file.

JohnNichols · ‎07-24-2022

it is 99 per year, not going to spend the money.

 module read_fis
    character *260 fisi
    character(:), allocatable :: str
    end module read_fis

    subroutine read_file_to_string
    use ifport
    use read_fis
    implicit none
    integer *4 :: iunit, istat, ilog
    integer *8 :: file_size, I,k,m,n
    character(len=1) :: c
    open(newunit = iunit, file = fisi, status = 'OLD', &
        form = 'UNFORMATTED', access = 'STREAM', iostat = istat)
    if (istat==0) then
        inquire(file=fisi, size=file_size )
        write(*,100)file_size
100     Format("File size is :: ",i10)

101     Format("File size is > :: ",i10," billion characters")

        ! write(*,100)file_size/1000
        !  write(*,100)file_size/1000000
        write(*,101)file_size/1000000000
        if (file_size > 0) then
            allocate(character(len=file_size ) :: str)
            read(iunit, pos=1, iostat = istat) str
            if (istat==0) then
                !make sure it was all read by trying to read more:
                read(iunit, pos = file_size+1, iostat = istat) c
                if (.not. IS_IOSTAT_END(istat)) then
                    write(*,*) 'Error: file was not completely read: ', trim(fisi)
                end if
            else
                write(*,*) 'Error reading file: ', trim(fisi)
            end if
            k = 1
            open(newunit = ilog, file = "ab.log", status = 'unknown') 
            do i = 1,200
    
                if(mod(1000,1) .eq. 0 ) then 
                !close(ilog)
               ! open(newunit = ilog, file = "ab1.log", status = 'unknown') 
                end if
                if(i .eq. 1) then
                m = k
                n = k + 56
                    write(ilog,1001)I,str(m:n)
                    m = n + 1
                    n = n + 46
                write(ilog,1002)I,str(m:n)
                m = m + 47
                n = m + 15
                write(ilog,1003)I,str(m:n)
                
                m = m + 16
                n = m + 49
                write(ilog,1004)I,str(m:n)
                m = m + 50
                n = m + 49
                write(ilog,1005)I,str(m:n)
                m = m + 50
                n = m + 49
                write(ilog,1006)I,str(m:n)
                
                m = m + 50
                n = m + 11
                write(ilog,1007)I,str(m:n)
                m = m + 12
                n = m + 11
                write(ilog,1008)I,str(m:n)
                
                m = m + 12
                n = m + 43
                
                else
                write(ilog,1002)I,str(m:n)
                    m = n + 1
                    n = n + 46
                write(ilog,1003)I,str(m:n)
                
                m = m + 16
                n = m + 49
                write(ilog,1004)I,str(m:n)
                m = m + 50
                n = m + 49
                write(ilog,1005)I,str(m:n)
                m = m + 50
                n = m + 49
                write(ilog,1006)I,str(m:n)
                
                m = m + 50
                n = m + 11
                write(ilog,1007)I,str(m:n)
                m = m + 12
                n = m + 11
                write(ilog,1008)I,str(m:n)
                
                m = m + 12
                n = m + 43
                end if               
1001            Format(I10,"  Line :: ", A57,\)
1002            Format(I10,"  Line :: ", A42)
1003            Format(I10,"  Line ::     ", A15,\)
1004            Format(I8,"  Line ::     ", A43)
1005            Format(I10,"  Line ::    ", A44)
1006            Format(I10,"  Line ::   ", A45)
1007            Format(I10,"  Line ::   ", (A),\)
1008            Format(I8,"  Line ::   ", (A)/)
            end do
            close(iunit, iostat=istat)
        else
            write(*,*) 'Error getting file size: ',  trim(fisi)
        end if
    else
        write(*,*) 'Error opening file: ', trim(fisi)
    end if

    return
    end

    !   Main program
    !===================================================================================

    use read_fis

    implicit none
    character(:), allocatable :: a
    character *260 fise
    character  *10 ext
    character   *1 cr
    character   *2 crlf

    integer *8 i, j, m, i1, i2, k
    integer *4 np, kp
    integer *4 :: iunit, istat

    cr = char(13)
    crlf = cr//char(10)

    ! call GETARG(1, fisi)
    !call GETARG(2, fise)
    fisi = "Test.stl"
    fise = "Test.out"


    kp = index(fise, '.', back = .true.) + 1
    ext = fise(kp:)

    call read_file_to_string ! str = fisi
    m = len(str) ! /2
    allocate(character(len=m) :: a)  !   a = fise

    open(newunit = iunit, file = fise, status = 'UNKNOWN', &
        form = 'UNFORMATTED', access = 'STREAM', iostat = istat)
    if(istat == 0) then
        continue
    else
        write(*,*) 'Error opening file: ', trim(fise)
        stop
    end if

    i = 1
    i1 = 1

    if(trim(ext) == 'pts' .or. trim(ext) == 'PTS') then
        np = 0
        do
            k = index(str(i:), 'vertex', kind = 8)
            if(k > 0) then
                np = np + 1
                i = i + k + int8(6)
                j = i + index(str(i:), cr, kind=8) - int8(2)
                ! j = i + index(str(i:), cr, kind =  - int8(2)
                i2 = i1 + j - i + int8(4)
                a(i1:i2) = str(i:j)//' 9'//crlf
                i1 = i2 + int8(1)
            else
                write(99,'(i0,t11,i0)') k, i
                write(iunit, pos = 1, iostat = istat) a(:i2-2)
                exit
            end if
        end do
    else
        do
            k = index(str(i:), 'vertex')
            if(k > 0) then
                i = i + k + 6
                j = i + index(str(i:), cr)
                i2 = i1 + j - i
                a(i1:i2) = str(i:j)
                i1 = i2 + 1
            else
                write(iunit, pos = 1, iostat = istat) a(:i2-2)
                exit
            end if
        end do
    end if

    deallocate(str)
    close(iunit)
    deallocate(a)

    stop
    end

I played with your code, str is very hard to take apart and even print.

Read one line at a time, find what you want and throw it away. That you should be able to write in an hour.

roy437 · ‎07-24-2022

Hooray,

I solved the problem, removed kind=8 and replaced index(str(i:) with index(str(i:i200) where i200=200. Execution time 11.70 sec. Thanks to everyone for the advice.

I7-3770, 16GB RAM, W10/64bit and ssd.

J. Roy

JohnNichols · ‎07-24-2022

11th Gen Intel(R) Core(TM) i7-11850H @ 2.50GHz 2.50 GHz

turn off the write to the file out and it runs in less than a second on 32 GB Win 11 and SAmsung SSD.

roy437 · ‎07-24-2022

Wow, probably the best time, congratulations!