I am attempting read a large data file into Fortran it contains about 40960000 rows, I later convert it to a matrix so it would be 6400 * 6400. When I try my code using a much smaller data set it works fine. However, it tells me that there is a segmentation fault, when I use my larger dataset. I suspect that it is a memory allocation issue. Is there a way to allocate more memory or otherwise correct this problem.
program infectgrid
implicit none
integer :: i, individual, nedge,&
tempinfect_tim_err, dumnum_infalpha0, dumnum_infalpha1, alpha0counter, &
alpha1counter, obsrand, minvalshift
character (LEN = 50) :: parameterfilelabel, parameterfile, networkfilelabel, infectfilenamelabel, networkfile, infectfilename
!allocatable to allow for reading in of individuals and time periods
integer, dimension(:) , allocatable :: xcord, ycord, dummy3, recover_tim1, recover_tim2, &
recover_tim3, recover_tim4, recover_tim5, sampleadjshort, shortsamplecount, &
cov1dum, covlog, covvect1uni
character, dimension (:), allocatable :: rec
integer, dimension (:, :) , allocatable :: link
real, dimension(:) , allocatable :: alphavect
call random_seed
open(12, file = "filenamesinfection.txt", status = "old")
read(12, *) parameterfile, networkfile, infectfilename
close(12)
open(12, file = parameterfile, status = "old")
read(12, *) individualname, alpha0name, alpha1name
read(12, *) individual, alpha0, alpha1
nedge = individual*individual
allocate(rec(1:nedge))
allocate(xcord(1:nedge))
allocate(ycord(1:nedge))
allocate(dummy3(1:nedge))
allocate(sampleadjshort(1:nedge))
allocate(shortsamplecount(1:nedge))
allocate(covlog(1:nedge))
allocate(cov1dum(1:nedge))
allocate(link(1:individual, 1:individual))
link = 0
covlog = 0
print *, individual, alpha0, alpha1
print *, link(individual, individual)
print *, "test"
open(10, file = networkfile, status = "old")
print *, networkfile
print *, "test"
do i = 1, nedge
read ( 10, *) rec(i), xcord(i), ycord(i), dummy3(i), cov1dum(i), sampleadjshort(i), shortsamplecount(i)
if (dummy3(i) == 1) then
link(xcord(i), ycord(i)) = 1
end if
if (xcord(i) == ycord(i)) then
covlog(i) = 1
end if
end do
close(10)
covvect1uni = pack(cov1dum, covlog == 1)
print *, "xcord", xcord(nedge), "ycord", ycord(nedge)
print *, link(individual, individual)
print *, "test"
The file prints out the first and second "test" but does not print out the link statement nor the line starting with "xcord" and soon after gives me a segmentation fault i.e. Segmentation fault nohup ./swnetworkinfectrecoveralpha05covgeo2.out
as to compiler options, I think that I am using most of what is available: ifort -free -O2 -stand f03 -check bounds -traceback -warn all -fstack-protector -assume protect_parens -implicitnone -o swnetworkinfectrecoveralpha05covgeo2.out swnetworkinfectrecoveralpha05covgeo2.f90
Related
I have 4 .mtx files that I am reading the values from. Two of them run perfectly when read from with no issues and produce the correct outputs into a .DAT file. However, the last 2 are extremely large files; it appears the code correctly reads from the files and runs, but I get no outputs and no errors when reading from these 2...not even the code timer prints the time. Any help is much appreciated! Here is the code:
program proj2matrixC40
implicit none
integer,parameter::dp=selected_real_kind(15,307)
! Set Global Variables
real(kind=dp), allocatable::Ax(:,:),A(:,:),Iglobal(:,:)
integer::At(1,3)
integer::nnz,w,n,k,ii,ff,kk
real(kind=dp)::t1,t2
call cpu_time(t1)
open(unit=78,file="e40r5000.mtx",status='old')
read(78,*) At
close(unit=78)
nnz = At(1,3)
n = At(1,1)
k = 40
kk = 35
allocate(Ax(nnz+1,3),A(nnz,3),Iglobal(k,k))
open(unit=61,file="e40r5000.mtx",status='old')
do w=1,nnz+1
read(61,*) Ax(w,:)
end do
open (unit = 53, file = "proj2matrixC40points.dat")
do ff=1,k
do ii=1,k
Iglobal(ii,ff) = (ii/ff)*(ff/ii)
end do
end do
A(1:nnz,:) = Ax(2:nnz+1,:)
call Arno(A)
call cpu_time(t2)
print '("Time elapsed = ",f10.8," seconds")', (t2 - t1)
contains
subroutine Arno(a)
real(kind=dp), intent(in)::a(:,:)
real(kind=dp),dimension(k,k)::H
real(kind=dp),dimension(k,k+1)::u,q,qconj
real(kind=dp),dimension(k,1)::x0
integer::j,f
call random_number(x0)
q(:,1) = x0(:,1)/norm2(x0(:,1))
do f=1,k
call spmat(a,q(:,f),u(:,f))
do j=1,f
qconj(j,:) = (q(:,j))
H(j,f) = dot_product(qconj(j,:),u(:,f))
u(:,f) = u(:,f) - H(j,f)*q(:,j)
end do
if (f.lt.k) then
H(f+1,f) = norm2(u(:,f))
if (H(f+1,f)==0) then
print *, "Matrix is reducible"
stop
end if
q(:,f+1) = u(:,f)/H(f+1,f)
end if
if (f==k) then
call qrit(H)
end if
end do
end subroutine
! QR Iteration with Shifts Subroutine
subroutine qrit(a)
real(kind=dp), intent(in)::a(:,:)
real(kind=dp)::sigmak
real(kind=dp),dimension(kk,k)::dia
real(kind=dp),dimension(k,k)::Qfinal,Rfinal,HH
real(kind=dp),dimension(k,k,kk)::H0,needQR
integer::v,z
HH = a
H0(:,:,1) = HH
do v=1,kk
sigmak = H0(k,k,v)
if (v-1==0) then
needQR(:,:,v) = HH - sigmak*Iglobal
else
needQR(:,:,v) = H0(:,:,v-1) - sigmak*Iglobal
end if
call givens2(needQR(:,:,v),Rfinal,Qfinal)
H0(:,:,v) = matmul(Rfinal,Qfinal) + sigmak*Iglobal
do z = 1,k
dia(v,z) = H0(z,z,v)
write(53,*) v," ", dia(v,z) ! Write values to .DAT file
end do
end do
end subroutine
! Sparse Matrix Vector Multiplication Subroutine
subroutine spmat(a,b,c)
real(kind=dp), intent(in)::a(:,:)
real(kind=dp), intent(in), dimension(k,1)::b
real(kind=dp), intent(out), dimension(k,1)::c
integer::m,rowi,columni
real(kind=dp), dimension(k,1)::x,y
x = b
y(:,1) = 0
do m = 1,nnz
rowi = a(m,1)
columni = a(m,2)
y(rowi,1) = y(rowi,1) + a(m,3)*x(columni,1)
end do
c(:,1) = y(:,1)
end subroutine
! QR Factorization Givens Rotations Subroutine
subroutine givens2(a,Rfinal,Qfinal)
real(kind=dp), intent(in)::a(:,:)
real(kind=dp), dimension(k,k,(k*k))::G,QQ
real(kind=dp), dimension(k,k), intent(out)::Rfinal,Qfinal
real(kind=dp), dimension(k,k)::I2,y,aa
real(kind=dp), dimension(1,k)::ek1,ek2
real(kind=dp)::c,s
integer::kt,m,nn,j,i,l,p
m = size(a,1)
nn = size(a,2)
aa = a
i = 1
do kt=1,nn-1
do j=m,kt+1,-1
if (aa(j,kt).eq.0) then
continue
else
ek1(1,:) = 0
ek2(1,:) = 0
do p=1,m
do l=1,m
I2(l,p) = (l/p)*(p/l)
end do
end do
c = aa(kt,kt)/sqrt(aa(kt,kt)**2 + aa(j,kt)**2)
s = aa(j,kt)/sqrt(aa(kt,kt)**2 + aa(j,kt)**2)
ek1(1,kt) = c
ek1(1,j) = s
ek2(1,kt) = -s
ek2(1,j) = c
I2(kt,:) = ek1(1,:)
I2(j,:) = ek2(1,:)
G(:,:,i) = I2
if (i.eq.1) then
QQ(:,:,i) = G(:,:,i)
else
QQ(:,:,i) = matmul(G(:,:,i),QQ(:,:,i-1))
end if
y = matmul(G(:,:,i),aa)
aa = y
if (kt.eq.nn-1) then
if (j.eq.kt+1) then
Qfinal = transpose(QQ(:,:,i))
Rfinal = aa
end if
end if
i = i + 1
end if
end do
end do
end subroutine
end program proj2matrixC40
A couple notes. The line which I put asterisks around (for this question) call mat_print('H',H) can't be deleted otherwise I get the wrong answers (this is strange...thoughts?). Also so your computer won't freeze opening the big files, their names are 'e40r5000.mtx' and 's3dkt3m2.mtx' (these are the two I have issues with). I am using gfortran version 8.1.0
Here is the link to the files
https://1drv.ms/f/s!AjG0dE43DVddaJfY62ABE8Yq3CI
When you need to add a call to a subroutine that shouldn't actually change anything in order to get things working, you probably have a memory corruption. This happens most often when you access arrays outside of their boundaries.
I have compiled it with some run time checks:
gfortran -o p2m -g -O0 -fbacktrace -fcheck=all -Wall proj2mat.f90
And it's already giving me some issues:
It's warning me about implicit type conversions. That shouldn't be too much of an issue if you trust your data.
In line 46 you have an array length mismatch (x0(:, 1) has length 40, q(:,1) is 41)
Similarly on line 108 (x=b) x is really large, but b is only 41 long.
I have stopped now, but I implore you to go through your code and clean it up. Use the compiler options above which will let you know when and where there is an array bound violation.
I'm trying to read a four column data file. However, because I'm having so much trouble, I'm just trying to read a single column of integers. Here is my code:
program RFF_Simple
implicit none
! Variables
character(len = 100):: line_in
character(len = :), allocatable :: filename
integer, dimension(:), allocatable :: weight,numbers
real, dimension(:), allocatable :: fm, fc
integer :: iostat_1, iostat_2
integer :: lun, length, index
! Body of RFF_Simple
! filename = 'data.txt'
filename = 'data_test.txt'
iostat_1 = 0
iostat_2 = 0
length = 0
open(newunit = lun, file = filename, status = 'old', iostat = iostat_1)
!Count how many lines are in the file (length)
if (iostat_1 == 0) then
do while(iostat_2 == 0)
read(lun, '(a)', iostat = iostat_2) line_in
if (iostat_2 == 0) then
length= length + 1
endif
enddo
endif
rewind(lun)
allocate(numbers(length)) !Allocate arrays to have same length as number of lines
iostat_1 = 0 !Reset
iostat_2 = 0
index = 1 !This whole thing is confusing so I don't know whether starting from 1 or 0 is better....
if (iostat_1 == 0) then
do while(iostat_2 == 0)
if(iostat_2 == 0) then
read(lun,*, iostat = iostat_2) numbers(index) !This crashes the program (Severe 408)
index = index + 1
endif
enddo
endif
write(*,*) 'Press Enter to Exit'
read(*,*)
end program RFF_Simple
The code compiles no problem, but running it yields the following: http://imgur.com/a/6ciJS
Yes I that is a print screen.
I don't even know where to start with this one.
The problem here is that you increment index after each successful read. After the last successful read we have index=length. You then add 1 to index and then attempt to read numbers(length+1) which results in a bounds violation. Rather than looping with a do while you can just use a regular do loop since we know the number of lines to read.
do index = 1, length
read(lun,*) numbers(index)
enddo
You could also test whether index is greater than length and bail out of the loop.
I have the following external file with 8 rows ad 11 columns. This file cannot be changed in anyway.
Name Sun Jupiter Saturn Uranus Neptune EarthBC Mercury Venus Mars Pluto
mass(Msun) 1.000 9.547922048e-4 2.858857575e-4 4.366245355e-5 5.151391279e-5 3.040433607e-6 1.660477111e-7 2.448326284e-6 3.227778035e-7 6.607313077e-9
a(AU) 5.20219308 9.54531447 19.19247127 30.13430686 1.00000159 0.38709889 0.72332614 1.52364259 39.80634014
e 0.04891224 0.05409072 0.04723911 0.00734566 0.01669714 0.20563613 0.00676922 0.09330305 0.25439724
I(deg) 1.30376425 2.48750693 0.77193683 1.77045595 0.00090235 7.00457121 3.39460666 1.84908137 17.12113756
M(deg) 240.35086842 045.76754755 171.41809349 293.26102612 094.81131358 242.19484206 345.30814403 330.93171908 024.68081529
w(deg) 274.15634048 339.60245769 098.79773610 255.50375800 286.84104687 029.14401042 054.54948603 286.56509772 114.39445491
OMEGA(deg) 100.50994468 113.63306105 073.98592654 131.78208581 176.14784451 048.32221297 076.66204037 049.53656349 110.32482041
This file is read by the following program which compiles properly
program readtable
implicit none
integer :: i, j, num_col, num_row
double precision, dimension (8,11) :: a
character(14), dimension (8) :: par
num_col = 4
num_row = 8
open(100,file='SSL.dat',status='old')
do j=1, num_row
read(100,*) par(j), (a(i,j), i=1,num_col)
end do
print *, par
print *, a(2,3) !Jupiter's Mass
end program
When I run this program as Fortran90 I get the following message:
At line 14 of file test.f (unit = 100, file='SSL.dat')
Fortran runtime error: Bad real number in item 2 of list input
I think I need to make a FORMAT() statement to help the program read the file properly but I can't seem to get the format right.
As agentp said list directed is fine here, you just have to account for the first 2 lines being different. I'd do it it something like (slight guess here - I'm not 100% convinced I understand what you want):
ian-admin#agon ~/test $ cat r.f90
Program readtable
Implicit None
Integer, Parameter :: wp = Selected_real_kind( 13, 70 )
Integer :: i, j, num_col, num_row
Real( wp ) :: msun
Real( wp ), Dimension (9,11) :: a
Character(14), Dimension (8) :: par
num_col = 9
num_row = 7
Open( 100, file = 'SSL.dat', status = 'old' )
Read( 100, * )
j = 1
Read( 100, * ) par(j), msun, (a(i,j), i=1,num_col)
Do j = 2, num_row
Read(100,*) par(j), (a(i,j), i=1,num_col)
End Do
Write( *, * ) par
Write( *, * ) a(2,3) !Jupiter's Mass
End Program readtable
ian-admin#agon ~/test $ gfortran -std=f2003 -Wall -Wextra -O -fcheck=all r.f90
ian-admin#agon ~/test $ ./a.out
mass(Msun) a(AU) e I(deg) M(deg) w(deg) OMEGA(deg)
5.4090720000000002E-002
I'm working on a project where I need to write some existing data to disk as ascii. I have something that works, but the IO itself is quite expensive and I'd like to optimise it further.
The data is basically an array of reals, however some of the columns store encoded strings which need to be recast as character strings (don't ask!). The input and output of this problem are beyond my control, I am receiving this real array and need to write it out as ascii.
I know that writing the array in one go as an unformatted write is faster, but this doesn't deal with the string columns correctly. Any ideas?
Here is some example code:
program test
implicit none
integer(kind=4), parameter :: nrows = 5000
integer(kind=4), parameter :: ncols = 400
integer, parameter :: real_kind = 8
integer(kind=4) :: i,j, handle
character(len=256) :: value_str
character(len=1) :: delimiter
real(kind=real_kind) :: data(nrows,ncols)
delimiter = " "
data(:,:) = 999.999
! Some examples of the "string columns"
data(:,10) = transfer(' foo ',data(1,1))
data(:,20) = transfer(' bar ',data(1,1))
handle=10
open(handle,file="out.txt",status="replace", access="stream")
do i=1,nrows
do j=1,ncols
! If this column contains encoded strings then recast
if((j==10).or.(j==20))then
write(handle) delimiter
value_str = transfer(data(i,j),value_str(1:real_kind))
write(handle) trim(value_str)
else
write(value_str,*) data(i,j)
write(handle) trim(value_str)
endif
enddo
write(handle) new_line('x')
enddo
close(handle)
end program test
gfortran test.F90 -o test.x
time test.x
real 0m2.65s
user 0m2.24s
sys 0m0.04s
Edit: removed "if(j/=1)" from original test.F90 code sample in response to comment.
Use the free formatting and have the system handle more for you. In this proposition, I handle the transfer beforehand and use a single loop to write the data to file. This is handy if you have only few columns of character data like the 2 in your example.
Your code will look like this
program test
implicit none
integer(kind=4), parameter :: nrows = 5000
integer(kind=4), parameter :: ncols = 400
integer, parameter :: real_kind = 8
integer, parameter :: pos1 = 10 ! I like named constants
integer, parameter :: pos2 = 20 ! I like named constants
integer(kind=4) :: i,j, handle
character(len=256) :: value_str
character(len=1) :: delimiter
real(kind=real_kind) :: data(nrows,ncols)
character(real_kind), dimension(nrows,2) :: cdata ! two columns array for
delimiter = " "
data(:,:) = 999.999
! Some examples of the "string columns"
data(:,pos1) = transfer(' foo ',data(1,1))
data(:,pos2) = transfer(' bar ',data(1,1))
handle=10
open(handle,file="out.txt",status="replace", form="formatted")
! Transfer beforehand
cdata(:,1) = transfer( data(:,pos1), cdata(1,1) )
cdata(:,2) = transfer( data(:,pos2), cdata(1,1) )
do i=1,nrows
write(handle,*) data(i,1:pos1-1), cdata(i,1)&
, data(i,pos1+1:pos2-1), cdata(i,2)&
, data(i,pos2+1:)
enddo
close(handle)
end program test
and give this timing
time ./test.x
real 0m1.696s
user 0m1.661s
sys 0m0.029s
instead of
time ./test.x
real 0m2.654s
user 0m2.616s
sys 0m0.032s
On my computer
I'm try to compile a Fortran application to write HDF5 files. My compiler is gfortran 4.7.2. Specifically, I'm trying to create a dataspace with a certain set of current dimensions and maximum dimensions. I want the last dimension to have an unlimited maximum dimension. There isn't much documentation for Fortran HDF5, but I was able to figure out that this can be specified by setting that dimension in question to H5S_UNLIMITED_F. This value is supposed to evaluate to -1, but in my application it instead evaluates to 0, which causes a runtime error because 0 is less than the corresponding current dimension (in my case, 20). This is the error:
HDF5-DIAG: Error detected in HDF5 (1.8.11) thread 0:
#000: H5S.c line 1388 in H5Screate_simple(): maxdims is smaller than dims
major: Invalid arguments to routine
minor: Bad value
I compiled one of the Fortran examples that came with HDF5 that uses the same H5S_UNLIMITED_F parameter (h5_extend.f90), but for that application, the parameter evaluates to -1 and there is no problem.
What might I be doing wrong?
Below is a test program I wrote to replicate the problem seen in my project:
program simple_test
use hdf5
implicit none
integer :: irank, hdferr
integer(hsize_t) :: ny, nx, nz
real, dimension(:,:,:), allocatable :: dset
character (len = 256) :: hdf_file, dlab
integer(hid_t) :: file_handle, mem_space, file_space, dset_handle
integer(hsize_t), dimension(:), allocatable :: dim_array, max_array
irank = 3
ny = 10
nx = 15
nz = 20
allocate (dset(ny, nx, nz))
hdf_file = 'simple_test.hdf5'
dlab = 'simple_data'
allocate (dim_array(irank))
allocate (max_array(irank))
dim_array = (/ ny, nx, nz /)
max_array = (/ ny, nx, H5S_UNLIMITED_F /)
print *, 'h5s_unlimited_f: ', h5s_unlimited_f
print *, 'dim_array: ', dim_array
print *, 'max_array: ', max_array
call h5open_f(hdferr)
if (hdferr .eq. -1) then
print *, 'Error opening HDF5 Fortran interface.'
end if
! Create a new file.
call h5fcreate_f(hdf_file, H5F_ACC_TRUNC_F, file_handle, hdferr)
if (hdferr .eq. -1) then
print *, 'Error creating HDF5 file.'
end if
! Create memory dataspace.
call h5screate_simple_f(irank, dim_array, mem_space, hdferr, max_array)
if (hdferr .eq. -1) then
print *, 'Error creating HDF5 memory dataspace.'
end if
! Create the dataset.
call h5dcreate_f(file_handle, trim(dlab), H5T_IEEE_F32LE, mem_space, &
dset_handle, hdferr)
if (hdferr .eq. -1) then
print *, 'Error creating HDF5 dataset.'
end if
! Create file dataspace.
call h5screate_simple_f(irank, dim_array, file_space, hdferr, max_array)
if (hdferr .eq. -1) then
print *, 'Error creating HDF5 file dataspace.'
end if
call h5dwrite_f(dset_handle, H5T_IEEE_F32LE, dset, dim_array, hdferr, &
mem_space, file_space)
if (hdferr .eq. -1) then
print *, 'Error writing HDF5 dataset.'
end if
call h5close_f(hdferr)
if (hdferr .eq. -1) then
print *, 'Error closing HDF5 Fortran interface.'
end if
deallocate (dset)
deallocate (dim_array)
deallocate (max_array)
end program simple_test
The first call to h5s_create_simple_f is what fails. If I change the memory dataspace to not use the max_array parameter (since it is optional and in my case perhaps unnecessary), then I still get the same error on the second call to h5s_create_simple_f.
I'm compiling as follows:
gfortran -c simple_test.f90 -o simple_test.o -I<hdf5_include_path>
gfortran -o simple_test simple_test.o -L<hdf5_lib_path> -lhdf5_fortran -lhdf5hl_fortran
I've also tried setting max_array(irank) to -1, but that yields an entirely different error.
(The original issue was that H5S_UNLIMITED_F is a variable that is initialised by a call to H5open_f, referencing it before that initialization is not permitted.)
Are you sure that the call to H5S_create_simple_f is what fails? Your traceback indicates an error from the dataset portion of the library.
I would expect a failure from dataset creation, because for variable sized datasets you need to specify the chunk size. Create a property list using H5Pcreate_f, then set the chunk size using H5Pset_chunk_f, then provide that property list after the error argument in the call to H5Dcreate_f. Comment if that doesn't make sense and I'll dig out an example.