I am using celerybeat to send periodic tasks to rabbitmq queue. It works as expected for some time and then send celery.backend_cleanup fails with SSL error.
[2016-10-24 04:00:02,309: DEBUG/MainProcess] Channel open
[2016-10-24 04:00:02,443: ERROR/MainProcess] Message Error: Couldn't apply scheduled task celery.backend_cleanup: [Errno 1] _ssl.c:1309: error:1409F07F:SSL routines:SSL3_WRITE_PENDING:bad write retry
[' File "/local/mnt/apps/ipcat/venvs/django16/bin/celery", line 9, in <module>\n load_entry_point(\'celery-ipcat==3.1.23\', \'console_scripts\', \'celery\')()\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/__main__.py", line 30, in main\n main()\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/celery.py", line 81, in main\n cmd.execute_from_commandline(argv)\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/celery.py", line 793, in execute_from_commandline\n super(CeleryCommand, self).execute_from_commandline(argv)))\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/base.py", line 311, in execute_from_commandline\n return self.handle_argv(self.prog_name, argv[1:])\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/celery.py", line 785, in handle_argv\n return self.execute(command, argv)\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/celery.py", line 717, in execute\n ).run_from_argv(self.prog_name, argv[1:], command=argv[0])\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/base.py", line 315, in run_from_argv\n sys.argv if argv is None else argv, command)\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/base.py", line 377, in handle_argv\n return self(*args, **options)\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/base.py", line 274, in __call__\n ret = self.run(*args, **kwargs)\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/bin/beat.py", line 79, in run\n return beat().run()\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/apps/beat.py", line 83, in run\n self.start_scheduler()\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/apps/beat.py", line 112, in start_scheduler\n beat.start()\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/beat.py", line 479, in start\n interval = self.scheduler.tick()\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/beat.py", line 221, in tick\n next_time_to_run = self.maybe_due(entry, self.publisher)\n', ' File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/beat.py", line 207, in maybe_due\n exc, traceback.format_stack(), exc_info=True)\n']
Traceback (most recent call last):
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/beat.py", line 204, in maybe_due
result = self.apply_async(entry, publisher=publisher)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/beat.py", line 260, in apply_async
entry, exc=exc)), sys.exc_info()[2])
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/beat.py", line 252, in apply_async
**entry.options)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/app/task.py", line 565, in apply_async
**dict(self._get_exec_options(), **options)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/app/base.py", line 354, in send_task
reply_to=reply_to or self.oid, **options
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/celery/app/amqp.py", line 310, in publish_task
**kwargs
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/messaging.py", line 172, in publish
routing_key, mandatory, immediate, exchange, declare)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/connection.py", line 436, in _ensured
return fun(*args, **kwargs)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/messaging.py", line 184, in _publish
[maybe_declare(entity) for entity in declare]
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/messaging.py", line 111, in maybe_declare
return maybe_declare(entity, self.channel, retry, **retry_policy)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/common.py", line 113, in maybe_declare
return _maybe_declare(entity, declared, ident, channel)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/common.py", line 120, in _maybe_declare
entity.declare()
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/entity.py", line 521, in declare
self.exchange.declare(nowait)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/kombu/entity.py", line 174, in declare
nowait=nowait, passive=passive,
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/amqp/channel.py", line 615, in exchange_declare
self._send_method((40, 10), args)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/amqp/abstract_channel.py", line 56, in _send_method
self.channel_id, method_sig, args, content,
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/amqp/method_framing.py", line 221, in write_method
write_frame(1, channel, payload)
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/amqp/transport.py", line 182, in write_frame
frame_type, channel, size, payload, 0xce,
File "/local/mnt/apps/ipcat/venvs/django16/lib/python2.7/site-packages/amqp/transport.py", line 254, in _write
n = write(s)
File "/usr/lib64/python2.7/ssl.py", line 172, in write
return self._sslobj.write(data)
SchedulingError: Couldn't apply scheduled task celery.backend_cleanup: [Errno 1] _ssl.c:1309: error:1409F07F:SSL routines:SSL3_WRITE_PENDING:bad write retry
[2016-10-24 04:00:02,743: DEBUG/MainProcess] beat: Waking up in 5.00 minutes.
After this failure all the subsequent send task fails with error as,
[2016-10-24 12:30:03,026: DEBUG/MainProcess] beat: Synchronizing schedule...
[2016-10-24 12:30:03,253: ERROR/MainProcess] Message Error: Couldn't apply scheduled task aggregate_hw_errata_sign_off_metrics_schedule: 'NoneType' object has no attribute 'do_handshake'
I could disable celery.backend_cleanjob using CELERY_TASK_RESULT_EXPIRES = None but I am afraid, it might fails on other tasks configured.
Any help or guidance is appreciated.
I think, I found the problem. I was creating connection and was passing it to exchange and queue instance. I modified the code to let Queue and Exchange create connection to broker. so far no issues after this change.
Thanks
Old code:
from kombu import Connection, Queue, Exchange
conn = Connection(
hostname=settings.BROKER_HOST,
port=settings.BROKER_PORT,
userid=settings.BROKER_USER,
password=settings.BROKER_PASSWORD,
virtual_host=settings.BROKER_VHOST,
connect_timeout=BROKER_CONNECTION_TIMEOUT,
ssl=BROKER_USE_SSL,
transport='pyamqp')
conn.connect()
channel = conn.channel()
exchange = Exchange('my.exchange', type='direct', passive=True, channel=channel)
CELERY_QUEUES = (
Queue('my.tasks', exchange=exchange, routing_key='my.metrics', channel=conn, passive=True),
)
New Code (fix):
from kombu import Queue, Exchange
exchange = Exchange('my.exchange', type='direct', passive=True)
CELERY_QUEUES = (
Queue('my.tasks', exchange=exchange, routing_key='my.metrics', passive=True),
)
Related
I am using this guide on the AWS deeplens. I am experimenting with the code and at one point it stopped working for the second standalone lambda function. At some point, the code started getting the exception and I only recently noticed it.
Here is the code in question:
def sendMessageToIoTTopic(iotMessage):
# this is a default topic if its not specified in the lambda envirionment
topicName = "worker-safety"
if "iot_topic" in os.environ:
topicName = os.environ['iot_topic']
print("Send message to topic: " + topicName + " before")
iotClient = boto3.client('iot-data', region_name='us-east-1')
response = iotClient.publish(
topic=topicName,
qos=1,
payload=json.dumps(iotMessage)
)
print("Send message to topic: " + topicName + "after")
print("using this topic **{}** response is {}".format(topicName, response))
The guide says to modify the default timeout from 1 second to 3 seconds. Here is the log for this code with a 3 second timeout.
2021-01-31T14:28:35.579-05:00 Before we send to iot MQTT?
2021-01-31T14:28:35.579-05:00 Send message to topic: worker-safety-demo-cloud before
2021-01-31T14:28:36.202-05:00 END RequestId: 89794691-a149-4480-b0bd-fd1a69522774
2021-01-31T14:28:36.202-05:00 REPORT RequestId: 89794691-a149-4480-b0bd-fd1a69522774 Duration: 3003.48 ms Billed Duration: 3000 ms Memory Size: 128 MB Max Memory Used: 35 MB
2021-01-31T14:28:36.202-05:00 2021-01-31T19:28:36.201Z 89794691-a149-4480-b0bd-fd1a69522774 Task timed out after 3.00 seconds
2021-01-31T14:28:38.016-05:00 START RequestId: 71562d34-28d6-471a-8a43-b1e0bab1d621 Version: $LATEST
Adjusting the timeout for the lambda function from 3 seconds to 15 seconds allows us to see the timeout error in detail. It shows that it is an SSL error.
This is the resultant log with more info:
2021-01-31T14:36:08.775-05:00 Before we send to iot MQTT?
2021-01-31T14:36:08.775-05:00 Send message to topic: worker-safety-demo-cloud before
2021-01-31T14:36:17.295-05:00 [ERROR] SSLError: SSL validation failed for https://data.iot.us-east-1.amazonaws.com/topics/worker-safety-demo-cloud?qos=1 [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1091) Traceback (most recent call last): File "/var/task/lambda_function.py", line 177, in lambda_handler sendMessageToIoTTopic(iotMessage) File "/var/task/lambda_function.py", line 119, in sendMessageToIoTTopic payload=json.dumps(iotMessage) File "/var/runtime/botocore/client.py", line 357, in _api_call return self._make_api_call(operation_name, kwargs) File "/var/runtime/botocore/client.py", line 663, in _make_api_call operation_model, request_dict, request_context) File "/var/runtime/botocore/client.py", line 682, in _make_request return self._endpoint.make_request(operation_model, request_dict) File "/var/runtime/botocore/endpoint.py", line 102, in make_request return self._send_request(request_dict, operation_model) File "/var/runtime/botocore/endpoint.py", line 137, in _send_request success_response, exception): File "/var/runtime/botocore/endpoint.py", line 256, in _needs_retry caught_exception=caught_exception, request_dict=request_dict) File "/var/runtime/botocore/hooks.py", line 356, in emit return self._emitter.emit(aliased_event_name, **kwargs) File "/var/runtime/botocore/hooks.py", line 228, in emit return self._emit(event_name, kwargs) File "/var/runtime/botocore/hooks.py", line 211, in _emit response = handler(**kwargs) File "/var/runtime/botocore/retryhandler.py", line 183, in __call__ if self._checker(attempts, response, caught_exception): File "/var/runtime/botocore/retryhandler.py", line 251, in __call__ caught_exception) File "/var/runtime/botocore/retryhandler.py", line 277, in _should_retry return self._checker(attempt_number, response, caught_exception) File "/var/runtime/botocore/retryhandler.py", line 317, in __call__ caught_exception) File "/var/runtime/botocore/retryhandler.py", line 223, in __call__ attempt_number, caught_exception) File "/var/runtime/botocore/retryhandler.py", line 359, in _check_caught_exception raise caught_exception File "/var/runtime/botocore/endpoint.py", line 200, in _do_get_response http_response = self._send(request) File "/var/runtime/botocore/endpoint.py", line 269, in _send return self.http_session.send(request) File "/var/runtime/botocore/httpsession.py", line 281, in send raise SSLError(endpoint_url=request.url, error=e)
2021-01-31T14:36:17.296-05:00 END RequestId: 0da11ea2-c9d3-42ce-a8d5-cd3e709bf42e
The SSL error in detail is this:
[ERROR] SSLError: SSL validation failed for https://data.iot.us-east-1.amazonaws.com/topics/worker-safety-demo-cloud?qos=1 [SSL: CERTIFICATE_VERIFY_FAILED]
certificate verify failed: unable to get local issuer certificate (_ssl.c:1091)Traceback (most recent call last):
File "/var/task/lambda_function.py", line 177, in lambda_handler sendMessageToIoTTopic(iotMessage)
File "/var/task/lambda_function.py", line 119, in sendMessageToIoTTopic payload=json.dumps(iotMessage)
File "/var/runtime/botocore/client.py", line 357, in _api_call return self._make_api_call(operation_name, kwargs)
File "/var/runtime/botocore/client.py", line 663, in _make_api_call operation_model, request_dict, request_context)
File "/var/runtime/botocore/client.py", line 682, in _make_request return self._endpoint.make_request(operation_model, request_dict)
File "/var/runtime/botocore/endpoint.py", line 102, in make_request return self._send_request(request_dict, operation_model)
File "/var/runtime/botocore/endpoint.py", line 137, in _send_request success_response, exception):
File "/var/runtime/botocore/endpoint.py", line 256, in _needs_retry caught_exception=caught_exception, request_dict=request_dict)
File "/var/runtime/botocore/hooks.py", line 356, in emit return self._emitter.emit(aliased_event_name, **kwargs)
File "/var/runtime/botocore/hooks.py", line 228, in emit return self._emit(event_name, kwargs)
File "/var/runtime/botocore/hooks.py", line 211, in _emit response = handler(**kwargs) File "/var/runtime/botocore/retryhandler.py", line 183, in __call__ if self._checker(attempts, response, caught_exception):
File "/var/runtime/botocore/retryhandler.py", line 251, in __call__ caught_exception) File "/var/runtime/botocore/retryhandler.py", line 277, in _should_retry return self._checker(attempt_number, response, caught_exception)
File "/var/runtime/botocore/retryhandler.py", line 317, in __call__ caught_exception) File "/var/runtime/botocore/retryhandler.py", line 223, in __call__ attempt_number, caught_exception)
File "/var/runtime/botocore/retryhandler.py", line 359, in _check_caught_exception raise caught_exception
File "/var/runtime/botocore/endpoint.py", line 200, in _do_get_response http_response = self._send(request)
File "/var/runtime/botocore/endpoint.py", line 269, in _send return self.http_session.send(request)
File "/var/runtime/botocore/httpsession.py", line 281, in send raise SSLError(endpoint_url=request.url, error=e)
I have implemented channel2 in my django project , after some time i start it i have received this error on my server , i have apache on ubuntu for my code.
2019-02-12 10:30:51,863 ERROR Exception inside application: [Errno -3] Temporary failure in name resolution
File "/var/www/RPG/python/lib/python3.6/site-packages/channels/consumer.py", line 59, in _call_
[receive, self.channel_receive], self.dispatch
File "/var/www/RPG/python/lib/python3.6/site-packages/channels/utils.py", line 59, in await_many_dispatch
await task
File "/var/www/RPG/python/lib/python3.6/site-packages/channels/utils.py", line 51, in await_many_dispatch
result = task.result()
File "/var/www/RPG/python/lib/python3.6/site-packages/channels_redis/core.py", line 429, in receive
real_channel
File "/var/www/RPG/python/lib/python3.6/site-packages/channels_redis/core.py", line 484, in receive_single
index, channel_key, timeout=self.brpop_timeout
File "/var/www/RPG/python/lib/python3.6/site-packages/channels_redis/core.py", line 324, in _brpop_with_clean
async with self.connection(index) as connection:
File "/var/www/RPG/python/lib/python3.6/site-packages/channels_redis/core.py", line 820, in _aenter_
self.conn = await self.pool.pop()
File "/var/www/RPG/python/lib/python3.6/site-packages/channels_redis/core.py", line 70, in pop
conns.append(await aioredis.create_redis(**self.host, loop=loop))
File "/var/www/RPG/python/lib/python3.6/site-packages/aioredis/commands/__init__.py", line 178, in create_redis
loop=loop)
File "/var/www/RPG/python/lib/python3.6/site-packages/aioredis/connection.py", line 108, in create_connection
timeout, loop=loop)
File "/usr/lib/python3.6/asyncio/tasks.py", line 339, in wait_for
return (yield from fut)
File "/var/www/RPG/python/lib/python3.6/site-packages/aioredis/stream.py", line 19, in open_connection
lambda: protocol, host, port, **kwds)
File "/usr/lib/python3.6/asyncio/base_events.py", line 739, in create_connection
infos = f1.result()
File "/usr/lib/python3.6/concurrent/futures/thread.py", line 56, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
[Errno -3] Temporary failure in name resolution
Here i attach a image which solve the problem but i need to find better solution.I don't think it's a properway to do this.
I got an error on my server.
DEBUG 2019-01-20 08:38:01,498 base 32461 140474717611392 Sending message of length 7076 to https://sentry.theinvaders.pro/api/30/store/
ERROR 2019-01-20 08:38:01,505 base 32461 140474190894848 Sentry responded with an error: <urlopen error [Errno -3] Temporary failure in name resolution> (url: https://sentry.theinvaders.pro/api/30/store/)
Traceback (most recent call last):
File "/usr/lib/python3.6/urllib/request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/usr/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/var/www/mass/var/venv/lib/python3.6/site-packages/raven/utils/http.py", line 31, in connect
timeout=self.timeout,
File "/usr/lib/python3.6/socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -3] Temporary failure in name resolution
>During handling of the above exception, another exception occurred:
>Traceback (most recent call last):
File "/var/www/mass/var/venv/lib/python3.6/site-packages/raven/transport/threaded.py", line 165, in send_sync
super(ThreadedHTTPTransport, self).send(url, data, headers)
File "/var/www/mass/var/venv/lib/python3.6/site-packages/raven/transport/http.py", line 43, in send
ca_certs=self.ca_certs,
File "/var/www/mass/var/venv/lib/python3.6/site-packages/raven/utils/http.py", line 66, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
'_open', req)
File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/var/www/mass/var/venv/lib/python3.6/site-packages/raven/utils/http.py", line 46, in https_open
return self.do_open(ValidHTTPSConnection, req)
File "/usr/lib/python3.6/urllib/request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno -3] Temporary failure in name resolution>
ERROR 2019-01-20 08:38:01,509 base 32461 140474190894848 ['OperationalError: could not translate host name "localhost" to address: Temporary failure in name resolution\n', ' File "django/core/handlers/exception.py", line 34, in inner', ' File "django/core/handlers/base.py", line 126, in _get_response', ' File "django/core/handlers/base.py", line 124, in _get_response', ' File "django/views/decorators/csrf.py", line 54, in wrapped_view', ' File "django/views/generic/base.py", line 68, in view', ' File "rest_framework/views.py", line 483, in dispatch', ' File "rest_framework/views.py", line 443, in handle_exception', ' File "rest_framework/views.py", line 480, in dispatch', ' File "rest_framework/decorators.py", line 53, in handler', ' File "mass/api/version1/candidate/views.py", line 66, in search_candidate', ' File "django/db/transaction.py", line 147, in __enter__', ' File "django/db/backends/base/base.py", line 378, in get_autocommit', ' File "django/db/backends/base/base.py", line 216, in ensure_connection', ' File "django/db/utils.py", line 89, in __exit__', ' File "django/db/backends/base/base.py", line 216, in ensure_connection', ' File "django/db/backends/base/base.py", line 194, in connect', ' File "django/db/backends/postgresql/base.py", line 178, in get_new_connection', ' File "psycopg2/__init__.py", line 130, in connect']
This lasted for 2 hours. Then the server rebooted. And everything began to work normally.
Please tell me what could be the cause of this.
The hosting provider had an error a DNS server.
I'm using Celery on Heroku with Redis as my broker. I've tried RabbitMQ as a broker as well, but keep getting the following error when trying to run a scheduled task:
Traceback (most recent call last):
File "/app/.heroku/python/lib/python2.7/site-packages/celery/beat.py", line 203, in maybe_due
result = self.apply_async(entry, publisher=publisher)
File "/app/.heroku/python/lib/python2.7/site-packages/celery/beat.py", line 259, in apply_async
entry, exc=exc)), sys.exc_info()[2])
File "/app/.heroku/python/lib/python2.7/site-packages/celery/beat.py", line 251, in apply_async
**entry.options)
File "/app/.heroku/python/lib/python2.7/site-packages/celery/app/task.py", line 555, in apply_async
**dict(self._get_exec_options(), **options)
File "/app/.heroku/python/lib/python2.7/site-packages/celery/app/base.py", line 347, in send_task
with self.producer_or_acquire(producer) as P:
File "/app/.heroku/python/lib/python2.7/site-packages/celery/app/base.py", line 402, in producer_or_acquire
producer, self.amqp.producer_pool.acquire, block=True,
File "/app/.heroku/python/lib/python2.7/site-packages/celery/app/amqp.py", line 492, in producer_pool
self.app.pool,
File "/app/.heroku/python/lib/python2.7/site-packages/celery/app/base.py", line 608, in pool
self._pool = self.connection().Pool(limit=limit)
File "/app/.heroku/python/lib/python2.7/site-packages/kombu/connection.py", line 612, in Pool
return ConnectionPool(self, limit, preload)
File "/app/.heroku/python/lib/python2.7/site-packages/kombu/connection.py", line 987, in __init__
preload=preload)
File "/app/.heroku/python/lib/python2.7/site-packages/kombu/connection.py", line 833, in __init__
self.setup()
File "/app/.heroku/python/lib/python2.7/site-packages/kombu/connection.py", line 1011, in setup
for i in range(self.limit):
SchedulingError: Couldn't apply scheduled task my_task: an integer is required
This is how my task is written:
#app.task(ignore_result=True)
def my_task():
do_something()
Any ideas what's going on?
It just occurred to me what was going on. In my settings file, I had the following line:
BROKER_POOL_LIMIT = os.environ.get('BROKER_POOL_LIMIT', 1)
I should have forced that to be an integer:
BROKER_POOL_LIMIT = int(os.environ.get('BROKER_POOL_LIMIT', 1))
I'm calling a list of urls from the same domain and returning a snip of their html for a few thousand domains but am getting this error about 1,000 rows or so in.
Is there anything I can do to avoid this error?
Does it make sense to create a wait step after every row? every few hundred rows?
Is there a better way to get around this?
File "/Users.../ap.py", line 144, in <module> simpleProg()
File "/Users.../ap.py", line 21, in simpleProg()
File "/Users.../ap.py", line 57, in first_step()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 400, in open
response = self._open(req, data)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 418, in _open
'_open', req)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 378, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1207, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1180, in do_open
r = h.getresponse(buffering=True)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1030, in getresponse
response.begin()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 407, in begin
version, status, reason = self._read_status()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 365, in _read_status
line = self.fp.readline()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 447, in readline
data = self._sock.recv(self._rbufsize)
socket.error: [Errno 54] Connection reset by peer