Python Kombu consumer get queue message but the callback can't trigger - kombu

If I run followed code, I found I can get message from the queue, but the callback can't triggered
from kombu.mixins import ConsumerMixin
from kombu import Exchange, Queue
task_exchange = Exchange('nginx', type='direct')
task_queues = [Queue(exchange=task_exchange, routing_key='nginx')]
class Worker(ConsumerMixin):
def __init__(self, connection):
self.connection = connection
def get_consumers(self, Consumer, channel):
return [Consumer(queues=task_queues,
callbacks=[self.task]
)]
def task(self, body, message):
print body
message.ack()
if __name__ == '__main__':
from kombu import Connection
with Connection('amqp://test:test#localhost:5672/test') as conn:
worker = Worker(conn)
worker.run()
I try to run python -m pdb test.py
170 -> def run(self, _tokens=1):
171 restart_limit = self.restart_limit
172 errors = (self.connection.connection_errors +
173 self.connection.channel_errors)
174 while not self.should_stop:
175 try:
(Pdb) l
176 if restart_limit.can_consume(_tokens):
177 for _ in self.consume(limit=None): # pragma: no cover
178 pass
179 else:
180 sleep(restart_limit.expected_time(_tokens))
181 except errors:
182 warn(W_CONN_LOST, exc_info=1)
It loop at
for _ in self.consume(limit=None): # pragma: no cover
pass

Runpython -m pdb test.py, Get in the connection.drain_events(), found the content.body encoding is binary,
if (content and
309 channel.auto_decode and
310 hasattr(content, 'content_encoding')):
311 -> try:
312 content.body = content.body.decode(content.content_encoding) #here get a error
313 except Exception:
314 pass
fix it
def get_consumers(self, Consumer, channel):
return [Consumer(queues=task_queues,
accept=['json', 'pickle'],
callbacks=[self.task]
)]

Related

InternalError when trying to predict with an endpoint on AWS

I use AWS SageMaker for a ML project and my TrainingJob for image classification has successfully finished. I use a separate inference.py for the inference, that looks like this:
import json
import logging
import sys
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models
import torchvision.transforms as transforms
import torchvision.transforms as T
from PIL import Image
import io
import requests
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))
# define model
def Net():
model = models.__dict__['resnet50'](pretrained=True)
for param in model.parameters():
param.requires_grad = False
num_features = model.fc.in_features
model.fc = nn.Sequential(
nn.Linear(num_features, int(num_features/2)),
nn.Linear(int(num_features/2), 5))
return model
# load model parameters
def model_fn(model_dir):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = Net().to(device)
with open(os.path.join(model_dir, "model.pth"), "rb") as f:
model.load_state_dict(torch.load(f))
model.eval()
return model
# deserialize input
def input_fn(request_body, content_type):
if content_type == 'image/jpeg':
img = Image.open(io.BytesIO(request_body))
return img
else:
raise ValueError("This model only supports jpeg input")
# inference
def predict_fn(input_object, model):
transform = T.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
])
input_object=transform(input_object)
input_object=input_object.unsqueeze(0)
with torch.no_grad():
prediction = model(input_object)
return prediction
I created a predictor and made a prediction like this:
from sagemaker.pytorch import PyTorchModel
from sagemaker.predictor import Predictor
model_location=estimator.model_data
pytorch_model = PyTorchModel(model_data=model_location,
role=sagemaker.get_execution_role(),
entry_point='inference.py',
py_version='py3',
framework_version='1.4')
predictor = pytorch_model.deploy(initial_instance_count=1, instance_type='ml.m5.large')
import io
with open("test1.jpg", "rb") as image:
img = image.read()
response=predictor.predict(img, initial_args={"ContentType": "image/jpeg"})
I got the following error message:
InternalFailure Traceback (most recent call last)
/tmp/ipykernel_7063/3976847146.py in <cell line: 1>()
----> 1 response=predictor.predict(img, initial_args={"ContentType": "image/jpeg"})
~/anaconda3/envs/python3/lib/python3.8/site-packages/sagemaker/predictor.py in predict(self, data, initial_args, target_model, target_variant, inference_id)
159 data, initial_args, target_model, target_variant, inference_id
160 )
--> 161 response = self.sagemaker_session.sagemaker_runtime_client.invoke_endpoint(**request_args)
162 return self._handle_response(response)
163
~/anaconda3/envs/python3/lib/python3.8/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
393 "%s() only accepts keyword arguments." % py_operation_name)
394 # The "self" in this scope is referring to the BaseClient.
--> 395 return self._make_api_call(operation_name, kwargs)
396
397 _api_call.__name__ = str(py_operation_name)
~/anaconda3/envs/python3/lib/python3.8/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
723 error_code = parsed_response.get("Error", {}).get("Code")
724 error_class = self.exceptions.from_code(error_code)
--> 725 raise error_class(parsed_response, operation_name)
726 else:
727 return parsed_response
InternalFailure: An error occurred (InternalFailure) when calling the InvokeEndpoint operation (reached max retries: 4): An exception occurred while sending request to model. Please contact customer support regarding request 34c901aa-76... .
I also tried this code in different accounts, but the issue remained.

ValidationException in Sagemaker pipeline creation

I am new to Sagmaker. I am creating a pipeline in sagemaker where I initialize the number of epochs as a pipeline parameter. But when I upsert, it shows this error.
Check the following code for reference, please.
epoch_count = ParameterInteger(name="EpochCount", default_value=1)
pipeline = Pipeline(
name=f"a_name",
parameters=[
training_instance_type,
training_instance_count,
epoch_count,
hugging_face_model_name,
endpoint_instance_type,
endpoint_instance_type_alternate,
],
steps=[step_train, step_register, step_deploy_lambda],
sagemaker_session=sagemaker_session,
)
Error - ---
---------------------------------------------------------------------------
ClientError Traceback (most recent call last)
<ipython-input-54-138a517611f0> in <module>
----> 1 pipeline.upsert(role_arn=role)
/opt/conda/lib/python3.7/site-packages/sagemaker/workflow/pipeline.py in upsert(self, role_arn, description, tags, parallelism_config)
217 """
218 try:
--> 219 response = self.create(role_arn, description, tags, parallelism_config)
220 except ClientError as e:
221 error = e.response["Error"]
/opt/conda/lib/python3.7/site-packages/sagemaker/workflow/pipeline.py in create(self, role_arn, description, tags, parallelism_config)
119 Tags=tags,
120 )
--> 121 return self.sagemaker_session.sagemaker_client.create_pipeline(**kwargs)
122
123 def _create_args(
/opt/conda/lib/python3.7/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
389 "%s() only accepts keyword arguments." % py_operation_name)
390 # The "self" in this scope is referring to the BaseClient.
--> 391 return self._make_api_call(operation_name, kwargs)
392
393 _api_call.__name__ = str(py_operation_name)
/opt/conda/lib/python3.7/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
717 error_code = parsed_response.get("Error", {}).get("Code")
718 error_class = self.exceptions.from_code(error_code)
--> 719 raise error_class(parsed_response, operation_name)
720 else:
721 return parsed_response
ClientError: An error occurred (ValidationException) when calling the CreatePipeline operation: Cannot assign property reference [Parameters.EpochCount] to argument of type [String]
I replace
epoch_count = ParameterInteger(name="EpochCount", default_value=1)
with
epoch_count = ParameterString(name="EpochCount", default_value="1")
And it works. Maybe we can only use an integer in pipeline parameters from the sagemaker notebook. But epoch_count is being used in the docker container, which is not directly something of Sagemaker, and that's my understanding.

Django queryset count() method raise "TypeError: unorderable types: NoneType() > int()"

My environment is Python3.5, Django1.8.3 and cx_Oracle5.3(They are checked by pip3 freeze).
Django query set raises a Type Error exception when count() method is called.
When it comes to Python2 + cx_oracle or Python3 + sqlite3 works fine without any exception but Python3 + cx_oracle.
Thue, I tried to update cx_Oracle version to 6.1(latest version) because I thought I could be some compatibility problem between cx_Oracle and Python3. However, It generates a different error.
I detail with the below code block, please refer it.
P.S: I Need to keep Django version to 1.8.3 for compatibility with my Apps.
models.py
from django.db import models
class Device(models.Model):
deviceClass = models.CharField(max_length=10)
class Meta:
db_table = 'TST_G2S_DEVICE'
cx_Oracle5.3
$ python3 manage.py shell
Python 3.5.2 (default, Nov 23 2017, 16:37:01)
Type "copyright", "credits" or "license" for more information.
IPython 2.4.1 -- An enhanced Interactive Python.
? -> Introduction and overview of IPython's features.
%quickref -> Quick reference.
help -> Python's own help system.
object? -> Details about 'object', use 'object??' for extra details.
In [1]: from polls.models import Device;
In [2]: dev = Device.objects.all()
In [3]: dev
Out[3]: []
In [4]: type(dev)
Out[4]: django.db.models.query.QuerySet
In [5]: dev.count()
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-5-72a7bdf9f7f7> in <module>()
----> 1 dev.count()
/usr/local/lib/python3.5/dist-packages/django/db/models/query.py in count(self)
316 return len(self._result_cache)
317
--> 318 return self.query.get_count(using=self.db)
319
320 def get(self, *args, **kwargs):
/usr/local/lib/python3.5/dist-packages/django/db/models/sql/query.py in get_count(self, using)
462 obj = self.clone()
463 obj.add_annotation(Count('*'), alias='__count', is_summary=True)
--> 464 number = obj.get_aggregation(using, ['__count'])['__count']
465 if number is None:
466 number = 0
/usr/local/lib/python3.5/dist-packages/django/db/models/sql/query.py in get_aggregation(self, using, added_aggregate_names)
443 outer_query.select_related = False
444 compiler = outer_query.get_compiler(using)
--> 445 result = compiler.execute_sql(SINGLE)
446 if result is None:
447 result = [None for q in outer_query.annotation_select.items()]
/usr/local/lib/python3.5/dist-packages/django/db/models/sql/compiler.py in execute_sql(self, result_type)
838 if result_type == SINGLE:
839 try:
--> 840 val = cursor.fetchone()
841 if val:
842 return val[0:self.col_count]
/usr/local/lib/python3.5/dist-packages/django/db/utils.py in inner(*args, **kwargs)
102 def inner(*args, **kwargs):
103 with self:
--> 104 return func(*args, **kwargs)
105 return inner
106
/usr/local/lib/python3.5/dist-packages/django/db/backends/oracle/base.py in fetchone(self)
507 if row is None:
508 return row
--> 509 return _rowfactory(row, self.cursor)
510
511 def fetchmany(self, size=None):
/usr/local/lib/python3.5/dist-packages/django/db/backends/oracle/base.py in _rowfactory(row, cursor)
575 # This comes from FloatField columns.
576 value = float(value)
--> 577 elif precision > 0:
578 # NUMBER(p,s) column: decimal-precision fixed point.
579 # This comes from IntField and DecimalField columns.
TypeError: unorderable types: NoneType() > int()
cx_Oracle6.1
Python 3.5.2 (default, Nov 23 2017, 16:37:01)
Type "copyright", "credits" or "license" for more information.
IPython 2.4.1 -- An enhanced Interactive Python.
? -> Introduction and overview of IPython's features.
%quickref -> Quick reference.
help -> Python's own help system.
object? -> Details about 'object', use 'object??' for extra details.
In [1]: from polls.models import Device;
In [2]: dev = Device.objects.all()
In [3]: dev
Out[3]: <repr(<django.db.models.query.QuerySet at 0x7f0ab12e8fd0>) failed: AttributeError: 'cx_Oracle.Cursor' object has no attribute 'numbersAsStrings'>
In [4]: type(dev)
Out[4]: django.db.models.query.QuerySet
In [5]: dev.count()
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-5-72a7bdf9f7f7> in <module>()
----> 1 dev.count()
/usr/local/lib/python3.5/dist-packages/django/db/models/query.py in count(self)
316 return len(self._result_cache)
317
--> 318 return self.query.get_count(using=self.db)
319
320 def get(self, *args, **kwargs):
/usr/local/lib/python3.5/dist-packages/django/db/models/sql/query.py in get_count(self, using)
462 obj = self.clone()
463 obj.add_annotation(Count('*'), alias='__count', is_summary=True)
--> 464 number = obj.get_aggregation(using, ['__count'])['__count']
465 if number is None:
466 number = 0
/usr/local/lib/python3.5/dist-packages/django/db/models/sql/query.py in get_aggregation(self, using, added_aggregate_names)
443 outer_query.select_related = False
444 compiler = outer_query.get_compiler(using)
--> 445 result = compiler.execute_sql(SINGLE)
446 if result is None:
447 result = [None for q in outer_query.annotation_select.items()]
/usr/local/lib/python3.5/dist-packages/django/db/models/sql/compiler.py in execute_sql(self, result_type)
825 return
826
--> 827 cursor = self.connection.cursor()
828 try:
829 cursor.execute(sql, params)
/usr/local/lib/python3.5/dist-packages/django/db/backends/base/base.py in cursor(self)
160 self.validate_thread_sharing()
161 if self.queries_logged:
--> 162 cursor = self.make_debug_cursor(self._cursor())
163 else:
164 cursor = self.make_cursor(self._cursor())
/usr/local/lib/python3.5/dist-packages/django/db/backends/base/base.py in _cursor(self)
135 self.ensure_connection()
136 with self.wrap_database_errors:
--> 137 return self.create_cursor()
138
139 def _commit(self):
/usr/local/lib/python3.5/dist-packages/django/db/backends/oracle/base.py in create_cursor(self)
260
261 def create_cursor(self):
--> 262 return FormatStylePlaceholderCursor(self.connection)
263
264 def _commit(self):
/usr/local/lib/python3.5/dist-packages/django/db/backends/oracle/base.py in __init__(self, connection)
417 self.cursor = connection.cursor()
418 # Necessary to retrieve decimal values without rounding error.
--> 419 self.cursor.numbersAsStrings = True
420 # Default arraysize of 1 is highly sub-optimal.
421 self.cursor.arraysize = 100
AttributeError: 'cx_Oracle.Cursor' object has no attribute 'numbersAsStrings'
Unfortunately for you, your restriction of staying on django 1.8 will not fix this issue.
This github issue describes the same situation you've encountered, with a very similar stack trace. A fix has been implemented in Django, but it apparently affects only 1.11 and forward releases, with no backport to 1.8 for the reason
This patch isn't completely compatible with cx_Oracle < 5.2, hence it
won't be backport to Django < 1.11

unpickle sklearn.tree.DescisionTreeRegressor in python 2 from python3

I wanna fit model in python 3.5 (numpy 1.11.2, sklearn 0.18.1)
import pickle
from sklearn.tree import DecisionTreeRegressor
clf = DecisionTreeRegressor()
X = np.array([[1,2,3,4],[1,1,2,2],[1,2,1,2]]).T
y = [1,1,0,0]
clf.fit(X,y)
with open(join(path_to_data, 'models', 'debug.model'), 'wb') as f:
pickle.dump(clf, f, protocol=2)
After pickling I try to unpickle model in python 2.7 (numpy 1.11.2, sklearn 0.18.1)
import pickle
with open(join(path_to_data, 'models', 'debug.model'), 'rb') as f:
clf = pickle.load(f)
but it raise error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-78-2eaf35b8e6d9> in <module>()
----> 1 joblib.load(join(path_to_data,'models','queryforest_debug.model'))
/home/iiivanitskiy/.local/lib/python2.7/site-packages/sklearn/externals/joblib/numpy_pickle.pyc in load(filename, mmap_mode)
573 return load_compatibility(fobj)
574
--> 575 obj = _unpickle(fobj, filename, mmap_mode)
576
577 return obj
/home/iiivanitskiy/.local/lib/python2.7/site-packages/sklearn/externals/joblib/numpy_pickle.pyc in _unpickle(fobj, filename, mmap_mode)
505 obj = None
506 try:
--> 507 obj = unpickler.load()
508 if unpickler.compat_mode:
509 warnings.warn("The file '%s' has been generated with a "
/usr/lib/python2.7/pickle.pyc in load(self)
856 while 1:
857 key = read(1)
--> 858 dispatch[key](self)
859 except _Stop, stopinst:
860 return stopinst.value
/home/iiivanitskiy/.local/lib/python2.7/site-packages/sklearn/externals/joblib/numpy_pickle.pyc in load_build(self)
325 NDArrayWrapper is used for backward compatibility with joblib <= 0.9.
326 """
--> 327 Unpickler.load_build(self)
328
329 # For backward compatibility, we support NDArrayWrapper objects.
/usr/lib/python2.7/pickle.pyc in load_build(self)
1215 setstate = getattr(inst, "__setstate__", None)
1216 if setstate:
-> 1217 setstate(state)
1218 return
1219 slotstate = None
ValueError: non-string names in Numpy dtype unpickling
Do we have the way to unpickle in python 2 DecisionTreeRegressor, which was pickled in python 3?

class 'psycopg2.InterfaceError': connection already closed

I am pulling my hair over this and am about ready to do something dirty.
I am running nginx + django + postgresql. Half the time I am trying to test my site and open a page, I get the following:
<class 'psycopg2.InterfaceError'> Python 2.7.2: /home/webapp/newavenue/bin/python
Fri Feb 24 08:28:58 2012
A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred.
/home/webapp/newavenue/lib/python2.7/site-packages/flup/server/fcgi_base.py in run(self=<flup.server.fcgi_base.Request object>)
572
573 try:
=> 574 protocolStatus, appStatus = self.server.handler(self)
575 except:
576 traceback.print_exc(file=self.stderr)
protocolStatus undefined, appStatus undefined, self = <flup.server.fcgi_base.Request object>, self.server = <flup.server.fcgi_fork.WSGIServer object>, self.server.handler = <bound method WSGIServer.handler of <flup.server.fcgi_fork.WSGIServer object>>
/home/webapp/newavenue/lib/python2.7/site-packages/flup/server/fcgi_base.py in handler(self=<flup.server.fcgi_fork.WSGIServer object>, req=<flup.server.fcgi_base.Request object>)
1157 try:
1158 try:
=> 1159 result = self.application(environ, start_response)
1160 try:
1161 for data in result:
result = None, self = <flup.server.fcgi_fork.WSGIServer object>, self.application = <django.core.handlers.wsgi.WSGIHandler object>, environ = {'CONTENT_LENGTH': '', 'CONTENT_TYPE': '', 'HTTP_ACCEPT': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'HTTP_ACCEPT_CHARSET': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'HTTP_ACCEPT_ENCODING': 'gzip,deflate,sdch', 'HTTP_ACCEPT_LANGUAGE': 'en-US,en;q=0.8', 'HTTP_CACHE_CONTROL': 'max-age=0', 'HTTP_CONNECTION': 'keep-alive', 'HTTP_COOKIE': 'csrftoken=8a0176b368b17af00a8563b5f95b134b; sess...e5bdd0d0a4e3092e7a9abd0029c02; django_language=en', 'HTTP_HOST': 'newavedev.zapto.org', ...}, start_response = <function start_response>
/home/webapp/newavenue/lib/python2.7/site-packages/django/core/handlers/wsgi.py in __call__(self=<django.core.handlers.wsgi.WSGIHandler object>, environ={'CONTENT_LENGTH': '', 'CONTENT_TYPE': '', 'HTTP_ACCEPT': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'HTTP_ACCEPT_CHARSET': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'HTTP_ACCEPT_ENCODING': 'gzip,deflate,sdch', 'HTTP_ACCEPT_LANGUAGE': 'en-US,en;q=0.8', 'HTTP_CACHE_CONTROL': 'max-age=0', 'HTTP_CONNECTION': 'keep-alive', 'HTTP_COOKIE': 'csrftoken=8a0176b368b17af00a8563b5f95b134b; sess...e5bdd0d0a4e3092e7a9abd0029c02; django_language=en', 'HTTP_HOST': 'newavedev.zapto.org', ...}, start_response=<function start_response>)
272 response = self.get_response(request)
273 finally:
=> 274 signals.request_finished.send(sender=self.__class__)
275
276 try:
global signals = <module 'django.core.signals' from '/home/webapp...python2.7/site-packages/django/core/signals.pyc'>, signals.request_finished = <django.dispatch.dispatcher.Signal object>, signals.request_finished.send = <bound method Signal.send of <django.dispatch.dispatcher.Signal object>>, sender undefined, self = <django.core.handlers.wsgi.WSGIHandler object>, self.__class__ = <class 'django.core.handlers.wsgi.WSGIHandler'>
/home/webapp/newavenue/lib/python2.7/site-packages/django/dispatch/dispatcher.py in send(self=<django.dispatch.dispatcher.Signal object>, sender=<class 'django.core.handlers.wsgi.WSGIHandler'>, **named={})
170
171 for receiver in self._live_receivers(_make_id(sender)):
=> 172 response = receiver(signal=self, sender=sender, **named)
173 responses.append((receiver, response))
174 return responses
response undefined, receiver = <function close_connection>, signal undefined, self = <django.dispatch.dispatcher.Signal object>, sender = <class 'django.core.handlers.wsgi.WSGIHandler'>, named = {}
/home/webapp/newavenue/lib/python2.7/site-packages/django/db/__init__.py in close_connection(**kwargs={'sender': <class 'django.core.handlers.wsgi.WSGIHandler'>, 'signal': <django.dispatch.dispatcher.Signal object>})
83 def close_connection(**kwargs):
84 for conn in connections.all():
=> 85 conn.close()
86 signals.request_finished.connect(close_connection)
87
conn = <django.db.backends.postgresql_psycopg2.base.DatabaseWrapper object>, conn.close = <bound method DatabaseWrapper.close of <django.d...postgresql_psycopg2.base.DatabaseWrapper object>>
/home/webapp/newavenue/lib/python2.7/site-packages/django/db/backends/__init__.py in close(self=<django.db.backends.postgresql_psycopg2.base.DatabaseWrapper object>)
242 def close(self):
243 if self.connection is not None:
=> 244 self.connection.close()
245 self.connection = None
246
self = <django.db.backends.postgresql_psycopg2.base.DatabaseWrapper object>, self.connection = <connection object at 0x1d37130; dsn: 'dbname=ne...er=postgres host=localhost port=5432', closed: 2>, self.connection.close = <built-in method close of psycopg2._psycopg.connection object>
<class 'psycopg2.InterfaceError'>: connection already closed
args = ('connection already closed',)
cursor = None
message = 'connection already closed'
pgcode = None
pgerror = None
The server is run with the following to generate the preceding flup traceback report:
python manage.py runfcgi host=127.0.0.1 port=8000 debug=False
Could I just get around this by patching in something like: "try: self.connection.close(), except: pass" around that offending line? It would probably work but I think it sounds dirty (messing with core) and I would like to try to find an alternative solution.
Edit: Considering this guy's approach too: commenting it out: http://osdir.com/ml/DjangoUsers/2009-04/msg01647.html
I think I got around the problem. This is not really a solution to the flup-psycopg2 setup but a sidestepping alternative. I switched from using fcgi to gunicorn and also wrapped the self.connection.close() line with "try: self.connection.close(), except: pass". This also solved a 502 Bad Gateway problem I was having with nginx.
Also, it seems my site runs a little faster, which is always a sweet side effect.