Trying to serialize numpy array with Pyro4 is returning the following type error
TypeError: don't know how to serialize class <type 'numpy.ndarray'>. Give it vars() or an appropriate __getstate__
The code is a the following
import numpy as np
import Pyro4
# set pickle serializer
Pyro4.config.SERIALIZERS_ACCEPTED = set(['pickle','json', 'marshal', 'serpent'])
#Pyro4.expose
class test(object):
def get_array(self):
return np.random.random((10,10))
def main():
# create a Pyro daemon
daemon = Pyro4.Daemon()
# register test instance
uri = daemon.register(test())
# print uri to connect to it in another console
print uri
# start the event loop of the server to wait for calls
daemon.requestLoop()
if __name__=="__main__":
main()
now open another console and try to call test instance doing the following
import Pyro4
Pyro4.config.SERIALIZERS_ACCEPTED = set(['pickle','json', 'marshal', 'serpent'])
# connect to URI which is printed above
# must be something like this 'PYRO:obj_c261949088104b839878255b98a9da90#localhost:57495'
p = Pyro4.Proxy(URI)
p.get_array()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python2.7/site-packages/Pyro4/core.py", line 171, in __call__
return self.__send(self.__name, args, kwargs)
File "/usr/local/lib/python2.7/site-packages/Pyro4/core.py", line 438, in _pyroInvoke
raise data
TypeError: don't know how to serialize class <type 'numpy.ndarray'>. Give it vars() or an appropriate __getstate
This is mentioned in the manual including what you can do to solve it: http://pythonhosted.org/Pyro4/tipstricks.html#pyro-and-numpy
In your code above, you didn't tell your client code to use pickle. You should use another config item for that (SERIALIZER). What you have there is meant for Pyro deamons instead.
Related
When I ran the below code,
import numpy as np
import tensorflow as tf
class Config:
activation = tf.nn.tanh
class Sample:
def function(self, x):
return self.config.activation(x)
def __init__(self, config):
self.config = config
if __name__ == "__main__":
with tf.Graph().as_default():
config = Config()
sample = Sample(config)
with tf.Session() as sess:
a = tf.constant(2.0)
print sess.run(sample.function(a))
I get this error message:
Traceback (most recent call last):
File "test.py", line 27, in <module>
print sess.run(sample.function(a))
File "test.py", line 11, in function
return self.config.activation(x)
File "/Users/byungwookang/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 2019, in tanh
with ops.name_scope(name, "Tanh", [x]) as name:
File "/Users/byungwookang/anaconda/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "/Users/byungwookang/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 4185, in name_scope
with g.as_default(), g.name_scope(n) as scope:
File "/Users/byungwookang/anaconda/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "/Users/byungwookang/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2839, in name_scope
if name:
File "/Users/byungwookang/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 541, in __nonzero__
raise TypeError("Using a `tf.Tensor` as a Python `bool` is not allowed. "
TypeError: Using a `tf.Tensor` as a Python `bool` is not allowed. Use `if t is not None:` instead of `if t:` to test if a tensor is defined, and use TensorFlow ops such as tf.cond to execute subgraphs conditioned on the value of a tensor.
In contrast, this code works as expected.
import numpy as np
import tensorflow as tf
class Config:
activation = np.tanh
class Sample:
def function(self, x):
return self.config.activation(x)
def __init__(self, config):
self.config = config
if __name__ == "__main__":
config = Config()
sample = Sample(config)
print sample.function(2.0)
print np.tanh(2.0)
It gives
0.964027580076
0.964027580076
I am curious why one can't pass a tensorflow built-in function as a variable (as done in the first code above), and whether there is a way to avoid the above error. Especially given the second code where a numpy function is passed nicely as a variable, it seems very strange to me that tensorflow doesn't allow this.
The reason your stuff does not work is because in your case
print sample.function # <bound method Sample.function of <__main__.Sample instance at 0xXXX>>
print tf.nn.tanh # <function tanh at 0xXXX>
are not the same, whereas in your second case they match. So when you run sample.function(a), not tanh but something else is executed.
It is hard for me to understand the purpose of all these classes and functions to do a simple job, so I just found the easiest way to modify whatever was written for it to work:
import numpy as np
import tensorflow as tf
def config():
return {'activation': tf.nn.tanh}
class Sample:
def function(self, x):
return self.config['activation'](x)
def __init__(self, config):
self.config = config
if __name__ == "__main__":
with tf.Graph().as_default(): # this is also not needed
sample = Sample(config())
with tf.Session() as sess:
a = tf.constant(2.0)
print sess.run(sample.function(a))
I am getting this output in the interactive mode.
class test:
def p(self):
print 'PP'
>>> f=open('E:\Python\Roy Progs\Test','w')
>>> t=test()
>>> import pickle
>>> pickle.dump(t,f)
>>> f.close()
>>> f=open('E:\Python\Roy Progs\Test','r')
>>> pickle.load(f).p()
PP
>>> f.close()
>>>
=============================== RESTART: Shell ===============================
>>> f=open('E:\Python\Roy Progs\Test','r')
>>> import pickle
>>> pickle.load(f).p()
Traceback (most recent call last):
File "<pyshell#14>", line 1, in <module>
pickle.load(f).p()
File "E:\Python\lib\pickle.py", line 1384, in load
return Unpickler(file).load()
File "E:\Python\lib\pickle.py", line 864, in load
dispatch[key](self)
File "E:\Python\lib\pickle.py", line 1075, in load_inst
klass = self.find_class(module, name)
File "E:\Python\lib\pickle.py", line 1132, in find_class
klass = getattr(mod, name)
AttributeError: 'module' object has no attribute 'test'
From the output I realize that the definition of the class (whose object is stored in the file) must be in there in the RAM at the time of retrieving data and using it. I however do not understand why this must be the case, by storing objects in the file am I not storing the class definition also?
The pickle module stores classes by named reference. If you change the name or location of the class pickle will raise an error.
A quick illustration of that can be seen in the interactive:
>>> class test:
x = 5
>>> from pickle import dumps
>>> dumps(test)
'c__main__\ntest\np0\n.' # pickle is storing a reference to 'test'
To successfully call load pickle must be able to find the previously defined class (which is destroyed when you call restart in idle)
I am new to Python and as my first project I am attempting to convert a Python2 script to Python3.
The script is failing when it attempts to serialize a class using pickle.
It seems as though it is failing as I am trying to save a class which uses the Cmd CLI.
This code works using Python2.
Can anyone tell me what is wrong with the script and how I fix it?
import sys
import cmd
try:
import pickle as pickle
except:
import pickle
import os.path
def main():
app = Labyrinth()
turnfile = "turn0.lwot"
app.Save(turnfile)
class CLI(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
class Labyrinth(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
def Save(self, fname):
with open(fname, 'wb') as f:
pickle.dump(self,f, 2)
f.close()
print ("Save Successful!")
sys.exit()
if __name__ == '__main__':
main()
Not all objects are picklable. In particular, file objects are problematic because you can't generally restore their state later. cmd.Cmd holds stdin and stdout file objects and that should make them unpicklable. I was quite surprised that it worked in python 2, but it didn't really... Even though the stdin and stdout pickled, the unpickled object you get back later doesn't work, as in this example:
>>> import sys
>>> import pickle
>>> sys.stdout.write('foo\n')
foo
>>> serialized = pickle.dumps(sys.stdout, 2)
>>> stdout = pickle.loads(serialized)
>>> stdout.write('bar\n')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ValueError: I/O operation on closed file
>>>
So, even though this bit of code didn't fail, the object shouldn't be usable later. You can add a few special methods to an object that let you fix objects so they can be serialized. Here, I've stripped the bad attributes on save and added them back on restore. Now you can pickle, unpickle and it actually works when you are done.
import sys
import cmd
try:
import cPickle as pickle
except:
import pickle
import os.path
def main():
app = Labyrinth()
turnfile = "turn0.lwot"
app.Save(turnfile)
class CLI(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
class Labyrinth(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
def Save(self, fname):
with open(fname, 'wb') as f:
pickle.dump(self,f, pickle.HIGHEST_PROTOCOL)
f.close()
print ("Save Successful!")
sys.exit()
def __getstate__(self):
# stdin/out are unpicklable. We'll get new ones on load
return tuple(((k,v) for k,v in self.__dict__.items()
if k not in ('stdin', 'stdout')))
def __setstate__(self, state):
self.__dict__.update(state)
self.stdin = sys.stdin
self.stdout = sys.stdout
if __name__ == '__main__':
main()
Playing with the protocol doesn't help. The full error message (which you should have included) is:
1027:~/mypy$ python3 stack41334887.py
Traceback (most recent call last):
File "stack41334887.py", line 33, in <module>
main()
File "stack41334887.py", line 14, in main
app.Save(turnfile)
File "stack41334887.py", line 27, in Save
pickle.dump(self,f, 3, fix_imports=True)
TypeError: cannot serialize '_io.TextIOWrapper' object
Python3 made some major changes in the io system. This TextIOWrapper is, I think new to Py3.
https://docs.python.org/3.1/library/io.html#io.TextIOWrapper
Can I use multiprocessing.Pool in a method of a class? also had problems serializing a TextIOWrapper.
=========
So inspireed by #tdelaney, I checked the stdin for my PY3 session:
In [1212]: sys.stdin
Out[1212]: <_io.TextIOWrapper name='<stdin>' mode='r' encoding='UTF-8'>
So that's the thing that can't be serialized.
I'm trying to use multiprocessing to speed up pandas excel reading. However when I use multiprocessing I'm getting the error
cPickle.PicklingError: Can't pickle : attribute lookup __builtin__.function failed
when I try to run the following:
import dill
from pathos.multiprocessing import ProcessPool
class A(object):
def __init__(self):
self.files = glob.glob(\*)
def read_file(self, filename):
return pd.read_excel(filename)
def file_data(self):
pool = ProcessPool(9)
file_list = [filename for filename in self.files]
df_list = pool.map(A().read_file, file_list)
combined_df = pd.concat(df_list, ignore_index=True)
Isn't pathos.multiprocessing designed to fix this issue? Am I overlooking something here?
Edit:
Full error code traces to
File "c:\users\zky3sse\appdata\local\continuum\anaconda2\lib\site-packages\pathos-0.2.0-py2.7.egg\
pathos\multiprocessing.py", line 136, in map
return _pool.map(star(f), zip(*args)) # chunksize
File "C:\Users\ZKY3SSE\AppData\Local\Continuum\Anaconda2\lib\multiprocessing\pool.py", line 251, in map
return self.map_async(func, iterable, chunksize).get()
File "C:\Users\ZKY3SSE\AppData\Local\Continuum\Anaconda2\lib\multiprocessing\pool.py", line 567, in get
raise self._value
It is possible that Pandas may be using Swig as a wrapper for C code. If this is the case, then dill may not work properly, and pathos would then switch to pickle. There are workarounds, as shown here: How to make my SWIG extension module work with Pickle?
Whenever I try to construct a string based on self.live_server_url, I get python TypeError messages. For example, I've tried the following string constructions (form 1 & 2 below), but I experience the same TypeError. My desired string is the Live Server URL with "/lists" appended. NOTE: the actual test does succeed to create a server and I can manually access the server, and more specifically, I can manually access the exact URL that I'm trying to build programmatically (e.g. 'http://localhost:8081/lists').
TypeErrors occur with these string constructions.
# FORM 1
lists_live_server_url = '%s%s' % (self.live_server_url, '/lists')
# FORM 2
lists_live_server_url = '{0}{1}'.format(self.live_server_url, '/lists')
self.browser.get(lists_live_server_url)
There is no python error with this form (nothing appended to string), albeit my test fails (as I would expect since it isn't accessing /lists).
self.browser.get(self.live_server_url)
Here is the python error that I'm getting.
/usr/local/Cellar/python3/3.4.2_1/Frameworks/Python.framework/Versions/3.4/bin/python3.4 /Applications/PyCharm.app/Contents/helpers/pycharm/django_test_manage.py test functional_tests.lists_tests.LiveNewVisitorTest.test_can_start_a_list_and_retrieve_it_later /Users/myusername/PycharmProjects/mysite_proj
Testing started at 11:55 AM ...
Creating test database for alias 'default'...
Traceback (most recent call last):
File "/usr/local/Cellar/python3/3.4.2_1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/wsgiref/handlers.py", line 137, in run
self.result = application(self.environ, self.start_response)
File "/usr/local/lib/python3.4/site-packages/django/test/testcases.py", line 1104, in __call__
return super(FSFilesHandler, self).__call__(environ, start_response)
File "/usr/local/lib/python3.4/site-packages/django/core/handlers/wsgi.py", line 189, in __call__
response = self.get_response(request)
File "/usr/local/lib/python3.4/site-packages/django/test/testcases.py", line 1087, in get_response
return self.serve(request)
File "/usr/local/lib/python3.4/site-packages/django/test/testcases.py", line 1099, in serve
return serve(request, final_rel_path, document_root=self.get_base_dir())
File "/usr/local/lib/python3.4/site-packages/django/views/static.py", line 54, in serve
fullpath = os.path.join(document_root, newpath)
File "/usr/local/Cellar/python3/3.4.2_1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/posixpath.py", line 82, in join
path += b
TypeError: unsupported operand type(s) for +=: 'NoneType' and 'str'
Am I unknowingly attempting to modify the live_server_url, which is leading to these TypeErrors? How could I programmatically build a string of live_server_url + "/lists"?
Here is the test that I am attempting...
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from django.test import LiveServerTestCase
class LiveNewVisitorTest(LiveServerTestCase):
def setUp(self):
self.browser = webdriver.Chrome()
self.browser.implicitly_wait(3)
def tearDown(self):
self.browser.close()
def test_can_start_a_list_and_retrieve_it_later(self):
#self.browser.get('http://localhost:8000/lists')
#self.browser.get('http://www.google.com')
#lists_live_server_url = '%s%s' % (self.live_server_url, '/lists')
#lists_live_server_url = '{0}{1}'.format(self.live_server_url, '/lists')
lists_live_server_url = self.live_server_url
self.browser.get(lists_live_server_url)
self.assertIn('To-Do', self.browser.title)
header_text = self.browser.find_element_by_tag_name('h1').text
self.assertIn('To-Do', header_text)
See this discussion on Reddit featuring the same error Traceback.
Basically, this is not a problem with anything within the Selenium tests but rather with your project's static file configuration.
From your question, I believe the key line within the Traceback is:
File "/usr/local/lib/python3.4/site-packages/django/views/static.py", line 54, in serve
fullpath = os.path.join(document_root, newpath)
This line indicates that an unsuccessful os.path.join is being attempted within django.views.static.
Set STATIC_ROOT in your project's settings.pyfile and you should be good.
Use StaticLiveServerTestCase instead may help