Haystack UnicodeDecodeError - django

Question

I'm using haystack and solr on Django. I get UnicodeDecodeError and I think the reason is the unicode data in the database are either converted to str or decoded to utf-8 which makes a later UnicodeDecodeError when sending via httplib or printing to the console.
I think the problem should before the haystack full_prepare function, since as I checked afterwards data is already ruined.
Any thoughts?
I setup.pyed install (haystack, pysolr, django)
and used the binary of lxml from http://www.lfd.uci.edu/~gohlke/pythonlibs/
Microsoft Windows [Version 6.1.7601]
Copyright (c) 2009 Microsoft Corporation. All rights reserved.
C:\Users\Pouria\Desktop\conference\conference>python manage.py rebuild_index
WARNING: This will irreparably remove EVERYTHING from your search index in connection 'default'.
Your choices after this are to restore from backups or rebuild via the `rebuild_index` command.
Are you sure you wish to continue? [y/N] y
Removing all documents from your index because you said so.
Failed to clear Solr index: [Reason: Error 404 Not Found]
All documents removed.
C:\Python27\lib\site-packages\django\db\models\fields\__init__.py:808: RuntimeWarning: DateTimeField
received a naive datetime (2013-02-09 08:47:27.110000) while time zone support is active.
RuntimeWarning)
Indexing 2 conferences.
ERROR:root:Error updating conferences using default
Traceback (most recent call last):
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 210, in handle_label
self.update_backend(label, using)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 256, in update_backend
do_update(backend, index, qs, start, end, total, self.verbosity)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 78, in do_update
backend.update(index, current_qs)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\backends\solr_backend.py", line 72, i
n update
self.conn.add(docs, commit=commit, boost=index.get_field_weights())
File "C:\Python27\lib\site-packages\pysolr.py", line 786, in add
return self._update(m, commit=commit, waitFlush=waitFlush, waitSearcher=waitSearcher)
File "C:\Python27\lib\site-packages\pysolr.py", line 379, in _update
return self._send_request('post', path, message, {'Content-type': 'text/xml; charset=utf-8'})
File "C:\Python27\lib\site-packages\pysolr.py", line 291, in _send_request
timeout=self.timeout)
File "C:\Python27\lib\site-packages\requests\api.py", line 87, in post
return request('post', url, data=data, **kwargs)
File "C:\Python27\lib\site-packages\requests\api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 279, in request
resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies
)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 373, in send
r = adapter.send(request, **kwargs)
File "C:\Python27\lib\site-packages\requests\adapters.py", line 171, in send
timeout=timeout
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 421, in url
open
body=body, headers=headers)
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 273, in _ma
ke_request
conn.request(method, url, **httplib_request_kw)
File "C:\Python27\lib\httplib.py", line 949, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 990, in _send_request
self.endheaders(body)
File "C:\Python27\lib\httplib.py", line 943, in endheaders
self._send_output(message_body)
File "C:\Python27\lib\httplib.py", line 810, in _send_output
self.send(message_body)
File "C:\Python27\lib\httplib.py", line 775, in send
self.sock.sendall(str)
File "C:\Python27\lib\socket.py", line 222, in meth
return getattr(self._sock,name)(*args)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 143-149: ordinal not in range(
128)
Traceback (most recent call last):
File "manage.py", line 10, in <module>
execute_from_command_line(sys.argv)
File "C:\Python27\lib\site-packages\django\core\management\__init__.py", line 443, in execute_from
_command_line
utility.execute()
File "C:\Python27\lib\site-packages\django\core\management\__init__.py", line 382, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 196, in run_from_argv
self.execute(*args, **options.__dict__)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 232, in execute
output = self.handle(*args, **options)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\rebuild_index.py"
, line 15, in handle
call_command('update_index', **options)
File "C:\Python27\lib\site-packages\django\core\management\__init__.py", line 150, in call_command
return klass.execute(*args, **defaults)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 232, in execute
output = self.handle(*args, **options)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 184, in handle
return super(Command, self).handle(*items, **options)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 341, in handle
label_output = self.handle_label(label, **options)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 210, in handle_label
self.update_backend(label, using)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 256, in update_backend
do_update(backend, index, qs, start, end, total, self.verbosity)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 78, in do_update
backend.update(index, current_qs)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\backends\solr_backend.py", line 72, i
n update
self.conn.add(docs, commit=commit, boost=index.get_field_weights())
File "C:\Python27\lib\site-packages\pysolr.py", line 786, in add
return self._update(m, commit=commit, waitFlush=waitFlush, waitSearcher=waitSearcher)
File "C:\Python27\lib\site-packages\pysolr.py", line 379, in _update
return self._send_request('post', path, message, {'Content-type': 'text/xml; charset=utf-8'})
File "C:\Python27\lib\site-packages\pysolr.py", line 291, in _send_request
timeout=self.timeout)
File "C:\Python27\lib\site-packages\requests\api.py", line 87, in post
return request('post', url, data=data, **kwargs)
File "C:\Python27\lib\site-packages\requests\api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 279, in request
resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies
)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 373, in send
r = adapter.send(request, **kwargs)
File "C:\Python27\lib\site-packages\requests\adapters.py", line 171, in send
timeout=timeout
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 421, in url
open
body=body, headers=headers)
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 273, in _ma
ke_request
conn.request(method, url, **httplib_request_kw)
File "C:\Python27\lib\httplib.py", line 949, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 990, in _send_request
self.endheaders(body)
File "C:\Python27\lib\httplib.py", line 943, in endheaders
self._send_output(message_body)
File "C:\Python27\lib\httplib.py", line 810, in _send_output
self.send(message_body)
File "C:\Python27\lib\httplib.py", line 775, in send
self.sock.sendall(str)
File "C:\Python27\lib\socket.py", line 222, in meth
return getattr(self._sock,name)(*args)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 143-149: ordinal not in range(
128)

Answer 1

I had the same error. It's seams to be fixed on master https://github.com/toastdriven/django-haystack. So you should use haystack 2.0 alpha/beta.