Error in using argparse to read data from Arduino using python - python-2.7

I am using windows 10 and I am reading ECG data using Arduino micro from pin A1. I used the serial plotter without any problems. The Arduino is connected to COM4
However, I want to read that ECG data using python and plot it there (interface). I came over a useful class called: deque. Link: http://electronut.in/plotting-real-time-data-from-arduino-using-python/
I am calling the function from another python file and I am getting this error:
usage: mac1.py [-h] --port PORT
mac1.py: error: argument --port is required
Code:
import serial
from Arduino import Arduino
import time
import Idr
import matplotlib.pyplot as plt
import matplotlib.animation as animation
if __name__ == '__main__':
Idr.main()
Idr.py function:
import sys, serial, argparse
import numpy as np
from time import sleep
from collections import deque
import matplotlib.pyplot as plt
import matplotlib.animation as animation
# plot class
class AnalogPlot:
# constr
def __init__(self, strPort, maxLen):
# open serial port
self.ser = serial.Serial(strPort, 9600)
self.ax = deque([0.0] * maxLen)
self.ay = deque([0.0] * maxLen)
self.maxLen = maxLen
# add to buffer
def addToBuf(self, buf, val):
if len(buf) < self.maxLen:
buf.append(val)
else:
buf.pop()
buf.appendleft(val)
# add data
def add(self, data):
assert (len(data) == 2)
self.addToBuf(self.ax, data[0])
self.addToBuf(self.ay, data[1])
# update plot
def update(self, frameNum, a1):
try:
line = self.ser.readline()
data = [float(val) for val in line.split()]
# print data
if (len(data) == 2):
self.add(data)
a1.set_data(range(self.maxLen), self.ay)
except KeyboardInterrupt:
print('exiting')
return a1,
# clean up
def close(self):
# close serial
self.ser.flush()
self.ser.close()
# main() function
def main():
# create parser
parser = argparse.ArgumentParser(description="LDR serial")
# add expected arguments
parser.add_argument('--port', dest='port', required=True)
# parse args
args = parser.parse_args()
# strPort = '/dev/tty.usbserial-A7006Yqh'
strPort = args.port
print('reading from serial port %s...' % strPort)
# plot parameters
analogPlot = AnalogPlot(strPort, 100)
print('plotting data...')
# set up animation
fig = plt.figure()
ax = plt.axes(xlim=(0, 100), ylim=(0, 1023))
a1, = ax.plot([], [])
anim = animation.FuncAnimation(fig, analogPlot.update,
fargs=(a1),
interval=50)
# show plot
plt.show()
# clean up
analogPlot.close()
print('exiting.')

Related

save and restore check point in convolutional neural network

i am try to save the training for my network in a checkpoint instead to trained each time i testing. So i don't know what is problem in my code when i run the test file, it going to train again. any body can help me plz ?
this is the train file
saver = tf.train.Saver()
with tf.Session(graph=graph) as session:
num_steps = 1001
session.run()
print('Initialized')
for step in range(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :, :, :]
batch_labels = train_labels[offset:(offset + batch_size), :]
print("batch_labels",batch_labels)
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction ], feed_dict=feed_dict)
if (step % 50 == 0):
print('Minibatch loss at step %d: %f' % (step, l))
print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))
print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_labels))
save_path = saver.save(session, "/home/owner//tensorflow/tensorflow/models/image/mnist/new_dataset/models.ckpt")
print("Model saved in file: %s" % save_path)
and here is the test file:
from __future__ import print_function
import numpy as np
import tensorflow as tf
from six.moves import cPickle as pickle
from six.moves import range
import time
from datetime import datetime
import tensorflow as tf
saver = tf.train.Saver()
init = tf.initialize_all_variables()
with tf.Session() as session:
saver.restore(session ,"/home/owner/tensorflow/tensorflow/models/image/mnist/new_dataset/models.ckpt")
print("Model restored.")
print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval() , test_labels, force = False ))

multiprocessing - pyodbc IOError: bad message length

I am unexpectedly getting IOError: bad message length error when trying to share pyodbc connection across multiple processes, especially when N is more than 4 (no. of cores). Sometimes I also get cPickle.UnpicklingError: invalid load key, '#'., pyodbc.ProgrammingError: ('24000', '[24000] [FreeTDS][SQL Server]Invalid cursor state (0) (SQLExecDirectW)') as errors.
# Import custom python packages
import multiprocessing
import multiprocessing.managers as mm
import pathos.multiprocessing as mp
import pyodbc, datetime, time
class MyConn(object):
def __init__(self):
self.conn = None
self.cursor = None
def connect_to_db(self):
self.conn = pyodbc.connect("DSN=cpmeast;UID=dntcore;PWD=dntcorevs2")
self.cursor = self.conn.cursor()
def run_qry(self, data):
print 'Running query', data
self.cursor.execute("WAITFOR DELAY '00:00:01';select GETDATE(), '"+str(data)+"';")
l = self.cursor.fetchall()
_l = []
for i in l:
_l.append(list(i))
print 'Result for query', data, _l
return _l
class MyManagerClass(object):
def __init__(self):
self.result = multiprocessing.Manager().list()
def read_data(self, *args):
conn = args[0][0]
data = args[0][1]
l = conn.run_qry(data)
self.result.append(l)
class MyManager(mm.BaseManager):
pass # Pass is really enough. Nothing needs to be done here.
def main():
time_start = time.time()
MyManager.register("MyConn", MyConn)
manager = MyManager()
manager.start()
a = manager.MyConn()
a.connect_to_db()
dbm = MyManagerClass()
pool = mp.ProcessingPool(4)
jobs = []
N = 5
for i in range(N):
jobs.append((a, str(i)))
for i in pool.imap(dbm.read_data, jobs):
print 'result'
pool.close()
pool.join()
print 'Result', dbm.result
print 'Closed'
time_stop = time.time()
msg = 'runtime: {0}'.format(str(datetime.timedelta
(seconds=time_stop-time_start)))
print msg
if __name__ == '__main__':
main()

Feature selection in scikit learn for multiple variables and thousands+ features

I am trying to perform feature selection for logistic regression classifier. Originally there are 4 variables: name, location, gender, and label = ethnicity. The three variables, namely the name, give rise to tens of thousands of more "features", for example, name "John Snow" will give rise to 2-letter substrings like 'jo', 'oh', 'hn'... etc. The feature set undergoes DictVectorization.
I am trying to follow this tutorial (http://scikit-learn.org/stable/auto_examples/feature_selection/plot_feature_selection.html) but I am not sure if I am doing it right since the tutorial is using a small number of features while mine has tens of thousands after vectorization. And also the plt.show() shows a blank figure.
# coding=utf-8
import pandas as pd
from pandas import DataFrame, Series
import numpy as np
import re
import random
import time
from random import randint
import csv
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from sklearn import svm
from sklearn.metrics import classification_report
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_selection import SelectPercentile, f_classif
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_curve
# Assign X and y variables
X = df.raw_name.values
X2 = df.name.values
X3 = df.gender.values
X4 = df.location.values
y = df.ethnicity_scan.values
# Feature extraction functions
def feature_full_name(nameString):
try:
full_name = nameString
if len(full_name) > 1: # not accept name with only 1 character
return full_name
else: return '?'
except: return '?'
def feature_avg_wordLength(nameString):
try:
space = 0
for i in nameString:
if i == ' ':
space += 1
length = float(len(nameString) - space)
name_entity = float(space + 1)
avg = round(float(length/name_entity), 0)
return avg
except:
return 0
def feature_name_entity(nameString2):
space = 0
try:
for i in nameString2:
if i == ' ':
space += 1
return space+1
except: return 0
def feature_gender(genString):
try:
gender = genString
if len(gender) >= 1:
return gender
else: return '?'
except: return '?'
def feature_noNeighborLoc(locString):
try:
x = re.sub(r'^[^, ]*', '', locString) # remove everything before and include first ','
y = x[2:] # remove subsequent ',' and ' '
return y
except: return '?'
def list_to_dict(substring_list):
try:
substring_dict = {}
for i in substring_list:
substring_dict['substring='+str(i)] = True
return substring_dict
except: return '?'
# Transform format of X variables, and spit out a numpy array for all features
my_dict13 = [{'name-entity': feature_name_entity(feature_full_name(i))} for i in X2]
my_dict14 = [{'avg-length': feature_avg_wordLength(feature_full_name(i))} for i in X]
my_dict15 = [{'gender': feature_full_name(i)} for i in X3]
my_dict16 = [{'location': feature_noNeighborLoc(feature_full_name(i))} for i in X4]
my_dict17 = [{'dummy1': 1} for i in X]
my_dict18 = [{'dummy2': random.randint(0,2)} for i in X]
all_dict = []
for i in range(0, len(my_dict)):
temp_dict = dict(my_dict13[i].items() + my_dict14[i].items()
+ my_dict15[i].items() + my_dict16[i].items() + my_dict17[i].items() + my_dict18[i].items()
)
all_dict.append(temp_dict)
newX = dv.fit_transform(all_dict)
# Separate the training and testing data sets
half_cut = int(len(df)/2.0)*-1
X_train = newX[:half_cut]
X_test = newX[half_cut:]
y_train = y[:half_cut]
y_test = y[half_cut:]
# Fitting X and y into model, using training data
lr = LogisticRegression()
lr.fit(X_train, y_train)
dv = DictVectorizer()
# Feature selection
plt.figure(1)
plt.clf()
X_indices = np.arange(X_train.shape[-1])
selector = SelectPercentile(f_classif, percentile=10)
selector.fit(X_train, y_train)
scores = -np.log10(selector.pvalues_)
scores /= scores.max()
plt.bar(X_indices - .45, scores, width=.2,
label=r'Univariate score ($-Log(p_{value})$)', color='g')
plt.show()
Warning:
E:\Program Files Extra\Python27\lib\site-packages\sklearn\feature_selection\univariate_selection.py:111: UserWarning: Features [[0 0 0 ..., 0 0 0]] are constant.
It looks like the way you split your data into training and testing sets is not working:
# Separate the training and testing data sets
X_train = newX[:half_cut]
X_test = newX[half_cut:]
If you already use sklearn, it is much more convenient to use the builtin splitting routine for this:
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.5, random_state=0)

Python Numpy : operands could not be broadcast together with shapes

I am getting this error "operands could not be broadcast together with shapes" for this code
import numpy as np
from sklearn.datasets import load_boston
from sklearn.linear_model import LinearRegression
beantown = load_boston()
x=beantown.data
y=beantown.target
model = LinearRegression()
model = model.fit(x,y)
def mse(truth, predictions):
return ((truth - predictions) ** 2).mean(None)
print model.score(x,y)
print mse(x,y)
The error is on the line print mse(x,y)
Error is ValueError:
operands could not be broadcast together with shapes (506,13) (506,)
Reshape y:
from sklearn.datasets import load_boston
from sklearn.linear_model import LinearRegression
beantown = load_boston()
x = beantown.data
y = beantown.target
y = y.reshape(y.size, 1)
model = LinearRegression()
model = model.fit(x, y)
def mse(truth, predictions):
return ((truth - predictions) ** 2).mean(None)
print model.score(x, y)
print mse(x, y)

Convert sample point to time values

import sys
import serial
import numpy as np
import matplotlib.pyplot as plt
from collections import deque
port = "COM11"
baud = 9600
timeout=1
ser = serial.Serial()
ser.port = port
ser.baudrate = baud
ser.timeout = timeout
a1 = deque([0.0]*100)
#ax = plt.axes(xlim=(0, 100), ylim=(0, 1000))
line, = plt.plot(a1)
plt.ion()
plt.ylim([0,1000])
try:
ser.open()
except:
sys.stderr.write("Error opening serial port %s\n" % (ser.portstr) )
sys.exit(1)
#ser.setRtsCts(0)
while 1:
# Read from serial port, blocking
data = ser.read(1)
# If there is more than 1 byte, read the rest
n = ser.inWaiting()
data = data + ser.read(n)
#sys.stdout.write(data)
print(a1)
a1.appendleft((data))
datatoplot = a1.pop()
line.set_ydata(a1)
plt.draw()
I am getting a plot between serial port values and sample points. I want to plot serial plot values vs time. Is there a way to convert sample points to time values, something like how to we convert sample point to frequency values using freqs = scipy.fftpack.fftfreq(n, d)
Thanks
If you want to plot the data against time from the start of the program, then:
import time
t0 = time.time()
tlist = deque([np.nan] * 100)
while 1:
# read the serial data ...
# when you have read a sample, capture the time difference
# and put it into a queue (similarly to the data values)
deltat = time.time() - t0
dlist.appendleft((deltat))
# remember to pop the data, as well
dlist.pop()
a1.pop()
# set the x and y data
line.set_xdata(tlist)
line.set_ydata(a1)
# draw it
plt.draw()
Now you have the number of seconds from the start of the program on the X axis.
If you want to have the real time shown, then use datetime.datetime objects:
import datetime
dlist = deque([datetime.datetime.now()] * 100)
while 1:
# capture the serial data ...
dilst.appendleft((datetime.datetime.now()))
# everything else as above
This should give you a plot with real time on the X axis.
import sys
import serial
import numpy as np
import matplotlib.pyplot as plt
import time
from collections import deque
from scipy import arange
port = "COM13"
baud = 9600
timeout=1
ser = serial.Serial()
ser.port = port
ser.baudrate = baud
ser.timeout = timeout
t0=time.time()
tlist = deque([np.nan]*10)
a1 = deque([0.0]*10)
#ax = plt.axes(xlim=(0, 100), ylim=(0, 1000))
line, = plt.plot(a1)
plt.ion()
plt.ylim([-100,100])
plt.grid(b=True,which= 'major' , color= 'g' , linestyle= '--')
#plt.grid(b=True,which= 'minor' , color= '-m' , linestyle= '--')
try:
ser.open()
except:
sys.stderr.write("Error opening serial port %s\n" % (ser.portstr) )
sys.exit(1)
#ser.setRtsCts(0)
while 1:
# Read from serial port, blocking
data = ser.read(1)
# If there is more than 1 byte, read the rest
n = ser.inWaiting()
data = data + ser.read(n)
#sys.stdout.write(data)
#print(a1)
#data1=int(data)-128
deltat = time.time() - t0
tlist.appendleft((deltat1))
datatoplot = tlist.pop()
a1.appendleft((data))
datatoplot = a1.pop()
line.set_xdata(tlist)
line.set_ydata(a1)
plt.hold(False)
plt.draw()
This is the complete code I used, and yes I had already changed that line.pop . But as I explained earlier in the comment I am not able to get the time values in x axis