Related
i cant get this code to work and im going crazy, i have some data(time vs current) and its a logarithmic graph, i want to use pchip interpolation or at least a cubic one between points as the data is real measurements and just assuming log gives too much error.
import os
import numpy as np
import pandas as pd
import sys
import xlrd
from scipy.interpolate import interp1d
df = pd.read_excel("re.xlsx")
array = df.values
ans = []
maxy = array.shape[0]
x = []
y = []
count = 0
while count < maxy:
x.append(float(array[count,0]))
y.append(float(array[count,1]))
count = count +1
print(x)
time = 0.001
while(time < 5):
f = interp1d(x, y, kind='cubic', copy=True, bounds_error=True, fill_value=np.nan)
print(time,f(time))
ans.append(f(time))
time = (time *1.1)
Here is the output, i dont get the error? the array is correct is it not? this is the cubic interpolation attempt but when i tried pchip i had a similar error saying x wast necessarily acceding
[0.0837, 0.0841, 0.0845, 0.0853, 0.0856, 0.0866, 0.0881, 0.0882, 0.09,
0.0921, 0.0921, 0.0947, 0.0973, 0.0977, 0.1042, 0.1122, 0.1202, 0.1233,
0.1304, 0.1365, 0.1415, 0.1432, 0.147, 0.1531, 0.1595, 0.1598, 0.1689,
0.1772, 0.1792, 0.191, 0.1999, 0.206, 0.2239, 0.2274, 0.2533, 0.2539, 0.2934,
0.294, 0.346, 0.3462, 0.4201, 0.4428, 0.5215, 0.5947, 0.6346, 0.7889, 0.8605,
0.9382, 0.9846, 1.128, 1.261, 1.4086, 1.5932, 1.6089, 1.8511, 2.0602, 2.167,
2.56, 2.6284, 3.228, 3.2321, 4.0363, 4.0959, 5.1183]
Traceback (most recent call last):
File "C:/Python27/projects/interpolationFromExcel.py", line 25, in <module>
f = interp1d(x, y, kind='cubic', copy=True, bounds_error=True,
fill_value=np.nan)
File "C:\Python27\lib\site-packages\scipy\interpolate\interpolate.py", line
535, in __init__
check_finite=False)
File "C:\Python27\lib\site-packages\scipy\interpolate\_bsplines.py", line
777, in make_interp_spline
raise ValueError("Expect x to be a 1-D sorted array_like.")
ValueError: Expect x to be a 1-D sorted array_like.
I seem to have some problems starting my learning... I am not sure why..
the network is multi input (72 1d arrays) and output is a 1d array length 24. the 1d array output consist of numbers related to 145 different classes.
So: 72 inputs => 24 outputs
Minimal working example - without the input/output being set.
import keras
from keras.utils import np_utils
from keras import metrics
from keras.models import Sequential
from keras.layers.core import Dense, Activation, Lambda, Reshape,Flatten
from keras.layers import Conv1D,Conv2D, MaxPooling2D, MaxPooling1D, Reshape, ZeroPadding2D
from keras.utils import np_utils
from keras.layers.advanced_activations import LeakyReLU, PReLU
from keras.layers.advanced_activations import ELU
from keras.models import Model
from keras.layers import Input, Dense
from keras.layers import Dropout
from keras import backend as K
from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import CSVLogger
from keras.callbacks import EarlyStopping
from keras.models import load_model
from keras.layers.merge import Concatenate
import numpy as np
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
nano_train_input = []
nano_train_output = []
nano_test_input = []
nano_test_output = []
## Creating train input:
for i in range(974):
nano_train_input.append(np.random.random((78,684,4)))
nano_train_output.append(np.random.randint(145,size=(228)).tolist())
## Creating test input:
for i in range(104):
nano_test_input.append(np.random.random((78,684,4)))
nano_test_output.append(np.random.randint(145,size=(228)).tolist())
def model(train_input, train_output, test_input, test_output, names=0):
# Paper uses dimension (40 x 45 =(15 * 3))
# Filter size 5
# Pooling size
# I use dimension (78 x 72 = (24 * 3)
# Filter size 9
print "In model"
i = 0
print_once = True
data_test_output = []
data_test_input = []
for matrix in test_input:
row,col,channel = matrix.shape
remove_output = (col/3)%24
remove_input = col%72
if remove_output > 0 :
test_output[i] = test_output[i][:-(remove_output)]
for split in chunks(test_output[i],24):
data_test_output.append(np.array(split))
if remove_input > 0:
out = np.split(matrix[:,:-(remove_input),:-1],matrix[:,:-(remove_input),:-1].shape[1]/72,axis=1)
else:
out = np.split(matrix[:,:,:-1],matrix[:,:,:-1].shape[1]/72,axis=1)
data_test_input.extend(out)
del out
i=i+1 # Increment
i=0
data_train_output = []
data_train_input = []
for matrix in train_input:
row,col,channel = matrix.shape
remove_output = (col/3)%24
remove_input = col%72
if remove_output > 0 :
train_output[i] = train_output[i][:-(remove_output)]
for split in chunks(train_output[i],24):
data_train_output.append(np.array(split))
if remove_input > 0:
out = np.split(matrix[:,:-(remove_input),:-1],matrix[:,:-(remove_input),:-1].shape[1]/72,axis=1)
else:
out = np.split(matrix[:,:,:-1],matrix[:,:,:-1].shape[1]/72,axis=1)
data_train_input.extend(out)
del out
i=i+1 # Increment
print
print "Len:"
print len(data_train_input)
print len(data_train_output)
print len(data_test_input)
print len(data_test_output)
print
print "Type[0]:"
print type(data_train_input[0])
print type(data_train_output[0])
print type(data_test_input[0])
print type(data_test_output[0])
print
print "Type:"
print type(data_train_input)
print type(data_train_output)
print type(data_test_input)
print type(data_test_output)
print
print "shape of [0]:"
print data_train_input[0].shape
print data_train_output[0].shape
print data_test_input[0].shape
print data_test_output[0].shape
list_of_input = [Input(shape = (78,3)) for i in range(72)]
list_of_conv_output = []
list_of_max_out = []
for i in range(72):
list_of_conv_output.append(Conv1D(filters = 32 , kernel_size = 6 , padding = "same", activation = 'relu')(list_of_input[i]))
list_of_max_out.append(MaxPooling1D(pool_size=3)(list_of_conv_output[i]))
merge = keras.layers.concatenate(list_of_max_out)
reshape = Flatten()(merge)
dense1 = Dense(units = 500, activation = 'relu', name = "dense_1")(reshape)
dense2 = Dense(units = 250, activation = 'relu', name = "dense_2")(dense1)
dense3 = Dense(units = 24 , activation = 'softmax', name = "dense_3")(dense2)
model = Model(inputs = list_of_input , outputs = dense3)
model.compile(loss="categorical_crossentropy", optimizer="adam" , metrics = [metrics.sparse_categorical_accuracy])
reduce_lr=ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, mode='auto', epsilon=0.01, cooldown=0, min_lr=0.000000000000000000001)
stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
print "Train!"
print model.summary()
hist_current = model.fit(x = ,
y = ,
shuffle=False,
validation_data=(,),
validation_split=0.1,
epochs=150000,
verbose=1,
callbacks=[reduce_lr,stop])
model(nano_train_input,nano_train_output,nano_test_input, nano_test_output)
The input and output is stored as a list of numpy.ndarrays.
This is a minimal working example.. how am I supposed to pass the input an output?
I would try:
merge = keras.layers.concatenate(list_of_max_out)
merge = Flatten()(merge) # or GlobalMaxPooling1D or GlobalAveragePooling1D
dense1 = Dense(500, activation = 'relu')(merge)
You probably want to apply something to transform your output from Convolutional layers. In order to do that - you need to squash the time / sequential dimension. In order to do that try techniques I provided.
If you take a look at your code and outputs you indeed have what you say: 24 outputs (data_train_outputs[0].shape). However, if you look at your layer output of Keras, you have this as output:
dense_3 (Dense) (None, 26, 145) 36395
I would say that this should be an array with shape (None, 24)....
I suggest you add a reshape layer to get the output you want to have!
I try to run this code
import sys
import numpy as np
filename = sys.argv[1]
X = []
y = []
with open(filename, 'r') as f:
for line in f.readlines():
xt, yt = [float(i) for i in line.split(',')]
X.append(xt)
y.append(yt)
and I get this error
4 filename = sys.argv[1]
5 X = []
6 y = []
IndexError: list index out of range
how can I fix it ?
I have a file in txt that I want to read my data from it.
4.94,4.37
-1.58,1.7
-4.45,1.88
-6.06,0.56
-1.22,2.23
-3.55,1.53
0.36,2.99
-3.24,0.48
1.31,2.76
2.17,3.99
2.94,3.25
-0.92,2.27
-0.91,2.0
1.24,4.75
1.56,3.52
-4.14,1.39
3.75,4.9
4.15,4.44
0.33,2.72
3.41,4.59
2.27,5.3
2.6,3.43
1.06,2.53
1.04,3.69
2.74,3.1
-0.71,2.72
-2.75,2.82
0.55,3.53
-3.45,1.77
1.09,4.61
2.47,4.24
-6.35,1.0
1.83,3.84
-0.68,2.42
-3.83,0.67
-2.03,1.07
3.13,3.19
0.92,4.21
4.02,5.24
3.89,3.94
-1.81,2.85
3.94,4.86
-2.0,1.31
0.54,3.99
0.78,2.92
2.15,4.72
2.55,3.83
-0.63,2.58
1.06,2.89
-0.36,1.99
Make sure you pass the filename as #hpaulj suggested. You could also check the length of sys.argv
import sys
print(sys.argv, len(sys.argv))
if len(sys.argv) < 2:
sys.exit('Usage: %s input_file' % sys.argv[0])
You may also want to check this helper class:
https://docs.python.org/2/library/fileinput.html#module-fileinput
I'm trying to run this code
from math import sqrt
import numpy as np
import warnings
from collections import Counter
import pandas as pd
import random
def k_nearest_neighbors(data,predict, k =3):
if len(data) >= k:
warnings.warn('K is set to a value less than total voting groups')
distances = []
for group in data:
for features in data[group]:
eucliden_distance = np.linalg.norm(np.array(features)-np.array(predict))
distances.append([eucliden_distance,group])
votes = [i[1] for i in sorted(distances)[:k]]
print(Counter(votes).most_common(1))
vote_result = Counter(votes).most_common(1)[0][0]
return vote_result
df = pd.read_csv('bc2.txt')
df.replace('?',-99999,inplace=True)
df.drop(['id'],1,inplace = True)
full_data = df.astype(float).values.tolist()
random.shuffle(full_data)
test_size = 0.2
train_set = {2:[],4:[]}
test_set = {2:[],4:[]}
train_data = full_data[:-int(test_size*len(full_data))]
test_data = full_data[-int(test_size*len(full_data)):]
for i in train_data:
train_set[i[-1]].append(i[:-1])
for i in train_data:
test_set[i[-1]].append(i[:-1])
correct = 0
total = 0
for group in test_set:
for data in test_set[group]:
vote = k_nearest_neighbors(train_set,data, k=5)
if group == vote:
correct += 1
total += 1
print ('Accuracy:',correct/total)
it comes out with this error msg
File "ml8.py", line 38, in <module>
train_set[i[-1]].append(i[:-1])
KeyError: 1.0
file m18.py is this above code file
below is the sample of txt file
id,clump_thickness,unif_cell_size,unif_cell_shape,marg_adhesion,single_epith_cell_size,bare_nuclei,bland_chrom,norm_nucleoli,mitoses,class
1000025,2,5,1,1,1,2,1,3,1,1
1002945,2,5,4,4,5,7,10,3,2,1
1015425,2,3,1,1,1,2,2,3,1,1
1016277,2,6,8,8,1,3,4,3,7,1
1017023,2,4,1,1,3,2,1,3,1,1
1017122,4,8,10,10,8,7,10,9,7,1
1018099,2,1,1,1,1,2,10,3,1,1
1018561,2,2,1,2,1,2,1,3,1,1
1033078,2,2,1,1,1,2,1,1,1,5
1033078,2,4,2,1,1,2,1,2,1,1
1035283,2,1,1,1,1,1,1,3,1,1
1036172,2,2,1,1,1,2,1,2,1,1
1041801,4,5,3,3,3,2,3,4,4,1
I'm using 2.7.11 version
Your train_set only contains keys 2 and 4, whereas your classes in that sample are 1 and 5.
Instead of using
train_set = {2:[],4:[]}
you might have better luck with defaultdict:
from collections import defaultdict
train_set = defaultdict(list)
This way a non-existent key will be initialized to a new empty list on first access.
import sys
import serial
import numpy as np
import matplotlib.pyplot as plt
from collections import deque
port = "COM11"
baud = 9600
timeout=1
ser = serial.Serial()
ser.port = port
ser.baudrate = baud
ser.timeout = timeout
a1 = deque([0.0]*100)
#ax = plt.axes(xlim=(0, 100), ylim=(0, 1000))
line, = plt.plot(a1)
plt.ion()
plt.ylim([0,1000])
try:
ser.open()
except:
sys.stderr.write("Error opening serial port %s\n" % (ser.portstr) )
sys.exit(1)
#ser.setRtsCts(0)
while 1:
# Read from serial port, blocking
data = ser.read(1)
# If there is more than 1 byte, read the rest
n = ser.inWaiting()
data = data + ser.read(n)
#sys.stdout.write(data)
print(a1)
a1.appendleft((data))
datatoplot = a1.pop()
line.set_ydata(a1)
plt.draw()
I am getting a plot between serial port values and sample points. I want to plot serial plot values vs time. Is there a way to convert sample points to time values, something like how to we convert sample point to frequency values using freqs = scipy.fftpack.fftfreq(n, d)
Thanks
If you want to plot the data against time from the start of the program, then:
import time
t0 = time.time()
tlist = deque([np.nan] * 100)
while 1:
# read the serial data ...
# when you have read a sample, capture the time difference
# and put it into a queue (similarly to the data values)
deltat = time.time() - t0
dlist.appendleft((deltat))
# remember to pop the data, as well
dlist.pop()
a1.pop()
# set the x and y data
line.set_xdata(tlist)
line.set_ydata(a1)
# draw it
plt.draw()
Now you have the number of seconds from the start of the program on the X axis.
If you want to have the real time shown, then use datetime.datetime objects:
import datetime
dlist = deque([datetime.datetime.now()] * 100)
while 1:
# capture the serial data ...
dilst.appendleft((datetime.datetime.now()))
# everything else as above
This should give you a plot with real time on the X axis.
import sys
import serial
import numpy as np
import matplotlib.pyplot as plt
import time
from collections import deque
from scipy import arange
port = "COM13"
baud = 9600
timeout=1
ser = serial.Serial()
ser.port = port
ser.baudrate = baud
ser.timeout = timeout
t0=time.time()
tlist = deque([np.nan]*10)
a1 = deque([0.0]*10)
#ax = plt.axes(xlim=(0, 100), ylim=(0, 1000))
line, = plt.plot(a1)
plt.ion()
plt.ylim([-100,100])
plt.grid(b=True,which= 'major' , color= 'g' , linestyle= '--')
#plt.grid(b=True,which= 'minor' , color= '-m' , linestyle= '--')
try:
ser.open()
except:
sys.stderr.write("Error opening serial port %s\n" % (ser.portstr) )
sys.exit(1)
#ser.setRtsCts(0)
while 1:
# Read from serial port, blocking
data = ser.read(1)
# If there is more than 1 byte, read the rest
n = ser.inWaiting()
data = data + ser.read(n)
#sys.stdout.write(data)
#print(a1)
#data1=int(data)-128
deltat = time.time() - t0
tlist.appendleft((deltat1))
datatoplot = tlist.pop()
a1.appendleft((data))
datatoplot = a1.pop()
line.set_xdata(tlist)
line.set_ydata(a1)
plt.hold(False)
plt.draw()
This is the complete code I used, and yes I had already changed that line.pop . But as I explained earlier in the comment I am not able to get the time values in x axis