Getting scipy to accept missing data - python-2.7

Question

I need to include missing data in the scipy pearsonr function. The problem is that if I read it from a file with e.g. 'NA' then I cannot convert it to float so that numpy will accept it. If I read it with numpy then the flexible array type with the 'NA' values cannot be used with 'dropna' or similar. How can I get scipy to accept missing data? I have read about data masks but I don't understand how to use it in the code.
Thanks,
#!/usr/bin/env python
import sys
import scipy.stats as sp
import numpy as np
f1=open(sys.argv[1],'r')
f2=open(sys.argv[2],'r')
g=open(sys.argv[3],'w')
f1.readline()
otus=[]
metanames=[]
result={}
for i in f1:
k1=i.split("\t")
k1[-1]=k1[-1].rstrip("\n")
otu=k1[0]
f2.seek(0)
result[otu]=[]
f2.readline()
for j in f2:
k2=j.split("\t")
k2[-1]=k2[-1].rstrip("\n")
if k2[0] not in metanames:
metanames.append(k2[0])
x=np.asarray(k1[1:])
y=np.asarray(k2[1:])
corr = sp.pearsonr(x, y)
result[otu].append(str(corr))
g.write("\t"+"\t".join(str(p) for p in metanames)+"\n")
for i in result.keys():
g.write(i+"\t"+"\t".join(str(p) for p in result[i][0])+"\n")
TypeError: cannot perform reduce with flexible type

Answer 1

I'm not sure I understand your problem. A minimal example
would have been helpful. But I guess you get something like the following:
>>> x = np.array([[ 1., 2., 3.], [ 4., 5., 6.], [ 7., 8., 9.]]).astype('object')
>>> x[0,1] = 'NA'
>>> x[2,2] = 'NA'
>>> x
array([[1.0, 'NA', 3.0],
[4.0, 5.0, 6.0],
[7.0, 8.0, 'NA']], dtype=object)
So you have a numpy array of type object because of t he 'NA's.
>>> x.astype('float')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ValueError: could not convert string to float: 'NA'
And that's why you can't convert it to float? So all you have
to do is replace the 'NA' with NaN
>>> x[x=='NA'] = np.nan
>>> x
array([[1.0, nan, 3.0],
[4.0, 5.0, 6.0],
[7.0, 8.0, nan]], dtype=object)
>>> x.astype('float')
array([[ 1., nan, 3.],
[ 4., 5., 6.],
[ 7., 8., nan]])

Answer 2

Ok, so I avoided using numpy at all and just manually removed 'nan' and corresponding value from both arrays.. by far easier and quicker, shoul dhave thought of it before.
S.
f1=open(sys.argv[1],'r')
f2=open(sys.argv[2],'r')
g=open(sys.argv[3],'w')
f1.readline()
otus=[]
metanames=[]
result={}
for i in f1:
k1=i.split("\t")
k1[-1]=k1[-1].rstrip("\n")
otu=k1[0]
if otu not in otus:
otus.append(otu)
f2.seek(0)
result[otu]=[]
f2.readline()
for j in f2:
k2=j.split("\t")
k2[-1]=k2[-1].rstrip("\n")
if k2[0] not in metanames:
metanames.append(k2[0])
x=k1[1:]
y=k2[1:]
c=-1
while c< len(x):
if x[c]=='NaN' or y[c]=='NaN':
del x[c]
del y[c]
else:
x[c]=float(x[c])
y[c]=float(y[c])
c=c+1
corr = sp.pearsonr(x, y)
result[otu].append(corr[0])
g.write("\t"+"\t".join(str(p) for p in metanames)+"\n")
for i in result.keys():
g.write(i)
for z in result[i]:
g.write("\t"+str(z))
g.write("\n")