from multiprocessing import Pool
def f(arg):
if arg == 1:
raise Exception("exception")
return "hello %s" % arg
p = Pool(4)
res = p.map_async(f,(1,2,3,4))
p.close()
p.join()
res.get()
Consider this contrived example where I am creating a process pool of 4 workers and assigning work in f(). My question was:
How can I retrieve the successful work that was done for arguments 2,3,4 (and at the same time do exception handling for argument 1) ?
As is the code just gives me:
Traceback (most recent call last):
File "process_test.py", line 13, in <module>
res.get()
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/multiprocessing/pool.py", line 567, in get
raise self._value
Exception: exception
You can also just do the error handling in the work function
def do_work(x):
try:
return (None, something_with(x))
except Exception as e:
return (e, None)
output = Pool(n).map(do_work, input)
for exc, result in output:
if exc:
handle_exc(exc)
else:
handle_result(result)
You can use the imap function.
iterator = p.imap(f, (1,2,3,4,5))
while True:
try:
print next(iterator)
except StopIteration:
break
except Exception as error:
print error
Related
I'm using tornado scheduler of apscheduler. Whenever a task is called, I need to log it's exceptions. For handling exceptions I've created a decorator that will get the future object and will take appropriate action. It's working fine, however it's not logging inside the callback function of future. I've done Pdb inside of the callback, and logger instance properties are as expected, still it's not logging in file at all.
Code is,
logger = logging.getLogger('myLogger')
def handle_callback(result):
logger.debug(result.result())
logger.info(result.result())
logger.error(result.result())
logger.exception(result.result())
logger.exception(result.exc_info())
def handle_exceptions():
def wrapped(fn):
#wraps(fn)
def wrapper(*args, **kwargs):
future = fn(*args, **kwargs)
future.add_done_callback(handle_callback)
return wrapper
return wrapped
#handle_exceptions()
#gen.coroutine
def run_task(job_id):
logger.info('executing job {}'.format(job_id))
raise MyException
P.S. I'm using python2.7
The wrapper is missing the return of the future - without this, ioloop won't continue if there would be some async call. Let's add some async call
#handle_exceptions
#gen.coroutine
def run_task(job_id):
logger.info('executing job {}'.format(job_id))
yield gen.sleep(1)
raise Exception('blah')
As you might noted, I've removed () from decorator to simplify it. It doesn't have to be nested. So the decorator could look like:
def handle_exceptions(fn):
#wraps(fn)
def wrapper(*args, **kwargs):
future = fn(*args, **kwargs)
future.add_done_callback(handle_callback)
return future # <<< we need this
return wrapper
Next, the handler's callback is calling Future.result() that will immediately re-raise exception. So it's better to check if there is an exception in the first place:
def handle_callback(result):
exc_info = result.exc_info()
if exc_info:
logger.error('EXCEPTION %s', exc_info)
Putting this together in simple example:
import logging
from functools import wraps, partial
from tornado import gen, ioloop
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def handle_callback(result):
exc_info = result.exc_info()
if exc_info:
logger.error('EXCEPTION %s', exc_info)
def handle_exceptions(fn):
#wraps(fn)
def wrapper(*args, **kwargs):
future = fn(*args, **kwargs)
future.add_done_callback(handle_callback)
return future
return wrapper
#handle_exceptions
#gen.coroutine
def run_task(job_id):
logger.info('executing job {}'.format(job_id))
yield gen.sleep(1)
raise Exception('blah')
ioloop.IOLoop.instance().run_sync(partial(run_task, 123))
Since the question does not provide any info about logging by itself, I've used the standard with changed level.
The code output:
INFO:root:executing job 123
ERROR:root:EXCEPTION (<type 'exceptions.Exception'>, Exception('blah',), <traceback object at 0x7f807df07dd0>)
Traceback (most recent call last):
File "test.py", line 31, in <module>
ioloop.IOLoop.instance().run_sync(partial(run_task, 123))
File "/tmp/so/lib/python2.7/site-packages/tornado/ioloop.py", line 458, in run_sync
return future_cell[0].result()
File "/tmp/so/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info)
File "/tmp/so/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
yielded = self.gen.send(value)
File "test.py", line 28, in run_task
raise Exception('blah')
Exception: blah
If there are any other issue, I presume it's related to the logging config/setup.
I am currently trying work on a project using Twisted Python, my problem specifically is my attempt to gain user input whilst also listening for connections using listenTCP(). I looked up the problem originally and found that stdio.StandardIO seems the most efficient way of doing so since I am already using Twisted. I have also seen the code examples found on twisted matrix, stdin.py and also stdiodemo.py however I am struggling with how to apply to example code to my specific problem given I need to read from the socket and also gather user input while performing tcp tasks.
The project I am working on is much larger however the small example code illustrates what I am trying to do and isolates the problem I am having. Any help in solving my problem is really appreciated.
Server.py
from twisted.internet.protocol import Factory
from twisted.protocols import basic
from twisted.internet import reactor, protocol, stdio
from Tkinter import *
import os, sys
class ServerProtocol(protocol.Protocol):
def __init__(self, factory):
self.factory = factory
stdio.StandardIO(self)
def connectionMade(self):
self.factory.numConnections += 1
self.factory.clients.append(self)
def dataReceived(self, data):
try:
print 'receiving data'
print data
except Exception, e:
print e
def connectionLost(self, reason):
self.factory.numConnections -= 1
self.factory.clients.remove(self)
class ServerFactory(Factory):
numConnections = 0
def buildProtocol(self, addr):
return ServerProtocol(self)
class StdioCommandLine(basic.LineReceiver):
from os import linesep as delimiter
def connectionMade(self):
self.transport.write('>>> ')
def lineReceived(self, line):
self.sendLine('Echo: ' + line)
self.transport.write('>>> ')
reactor.listenTCP(9001, ServerFactory())
stdio.StandardIO(StdioCommandLine())
reactor.run()
Client.py
from twisted.internet import reactor, protocol
import os, time, sys
import argparse
class MessageClientProtocol(protocol.Protocol):
def __init__(self, factory):
self.factory = factory
def connectionMade(self):
self.sendMessage()
def sendMessage(self):
print 'sending message'
try:
self.transport.write('hello world')
except e as Exception:
print e
def dataReceived(self, data):
print 'received: ', data
self.sendMessage()
class MessageClientFactory(protocol.ClientFactory):
def __init__(self, message):
self.message = message
def buildProtocol(self, addr):
return MessageClientProtocol(self)
def clientConnectionFailed(self, connector, reason):
print 'Connection Failed: ', reason.getErrorMessage()
reactor.stop()
def clientConnectionLost(self, connector, reason):
print 'Connection Lost: ', reason.getErrorMessage()
reactor.connectTCP('192.168.1.70', 9001, MessageClientFactory('hello world - client'))
reactor.run()
At the moment the above code is returning an Unhanded Error as follows. This demonstrates me using stdin, then it returning the data to stdout and a client the connects causing the error:
python Server.py
>>> hello
Echo: hello
>>> Unhandled Error Traceback (most recent call last):
File
"/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/twisted/python/log.py",
line 84, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File
"/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/twisted/python/context.py",
line 118, in callWithContext return
self.currentContext().callWithContext(ctx, func, *args, **kw)
File
"/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/twisted/python/context.py",
line 81, in callWithContext
return func(*args,**kw)
File
"/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/twisted/internet/selectreactor.py",
line 149, in _doReadOrWrite
why = getattr(selectable, method)()
--- exception caught here ---
File
"/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/twisted/internet/tcp.py",
line 1067, in doRead
protocol = s
The traceback you provided seems to be cut off. I tried to run the code on my machine and it shows this traceback:
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/twisted/python/log.py", line 84, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/lib/python2.7/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python2.7/site-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/usr/lib/python2.7/site-packages/twisted/internet/posixbase.py", line 597, in _doReadOrWrite
why = selectable.doRead()
--- <exception caught here> ---
File "/usr/lib/python2.7/site-packages/twisted/internet/tcp.py", line 1067, in doRead
protocol = self.factory.buildProtocol(self._buildAddr(addr))
File "Server.py", line 30, in buildProtocol
return ServerProtocol(self)
File "Server.py", line 10, in __init__
stdio.StandardIO(self)
File "/usr/lib/python2.7/site-packages/twisted/internet/_posixstdio.py", line 42, in __init__
self.protocol.makeConnection(self)
File "/usr/lib/python2.7/site-packages/twisted/internet/protocol.py", line 490, in makeConnection
self.connectionMade()
File "Server.py", line 14, in connectionMade
self.factory.clients.append(self)
exceptions.AttributeError: ServerFactory instance has no attribute 'clients'
As can be easily seen with the full traceback, the factory is missing the client attribute. This can be fixed e.g. by adding this to your ServerFactory class:
def __init__(self):
self.clients = []
Here is my code :-
import xml.etree.ElementTree as ET
filename= "kolkata_india.osm"
def count_tags(filename):
tags = {}
for event, elem in ET.iterparse(filename):
if elem.tag in tags:
tags[elem.tag] += 1
else:
tags[elem.tag] = 1
print tags
if __name__ == "__main__":
count_tags(filename)
and here is the error message :-
C:\Python27\Udacity\mongodb>python mapparser.py
Traceback (most recent call last):
File "mapparser.py", line 17, in <module>
count_tags(filename)
File "mapparser.py", line 8, in count_tags
filename = kolkata_india.osm
NameError: global name 'kolkata_india' is not defined
What is it here that I am doing wrong ?
I have a pika consumer
when i run it and sends SIGHUP signal, It gives me an exception
Consumertest.py
import signal
import traceback
import pika
from time import sleep
received_signal = False
def signal_handler(signal, frame):
global received_signal
received_signal = True
exit(1)
def sighup_handler(signal, frame):
print "sighup received"
signal.signal(signal.SIGHUP, sighup_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
mq_server = "localhost"
mq_exchange = "my_exchange"
my_queue = "test_queue"
try:
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(mq_server))
except:
exit(1)
mq_channel = mq_connection.channel()
mq_channel.exchange_declare(exchange=mq_exchange, durable=True)
mq_channel.queue_declare(queue=my_queue, durable=True)
mq_channel.queue_bind(my_queue, mq_exchange)
def callback(ch, method, properties, body):
try:
sleep(10)
ch.basic_reject(delivery_tag=method.delivery_tag)
except Exception as e:
traceback.print_exc() # region Message consumption
try:
print ' [*] Waiting for messages. To exit press CTRL+C'
mq_channel.basic_consume(callback, queue=my_queue)
mq_channel.start_consuming()
except Exception as e:
traceback.print_exc()
while True:
pass
exception:
[*] Waiting for messages. To exit press CTRL+C
sighup received
Traceback (most recent call last):
File "/run/media/bluto/04D0CF8ED0CF8500/Email_Projects/new_email_workers/testsigHup.py", line 47, in <module>
mq_channel.start_consuming()
File "/usr/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 814, in start_consuming
self.connection.process_data_events()
File "/usr/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 168, in process_data_events
if self._handle_read():
File "/usr/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 271, in _handle_read
if self._read_poller.ready():
File "/usr/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 54, in ready
events = self.poller.poll(self.poll_timeout)
error: (4, 'Interrupted system call')
I have a cronjob which processes emails and stores them in the system. It'd been working well for over a year but now all of a sudden it has started giving these weird transaction errors randomly every now and then.
2010-09-02-01:45:11 ERROR Exception occured during mail processing
Traceback (most recent call last):
File "/var/www/example.com/project/scripts/cronjobs.py", line 453, in process_msg
source, email, 'i' if rejection_reason else 'v', rejection_reason)
File "/usr/lib/python2.5/site-packages/django/db/transaction.py", line 267, in _commit_manually
leave_transaction_management()
File "/usr/lib/python2.5/site-packages/django/db/transaction.py", line 77, in leave_transaction_management
raise TransactionManagementError("Transaction managed block ended with pending COMMIT/ROLLBACK")
TransactionManagementError: Transaction managed block ended with pending COMMIT/ROLLBACK
It happens without any reason and there isn't any other context information available to me. Here is the structure of my process_msg function:
#commit_manually
def save_email( u,l, d, t, s, e, status, reason):
try:
# ... bla bla bla ...
commit()
exception, e:
rollback()
raise e
def process_msg(m):
try:
#....
save_email(u, l, d, t
source, email, 'i' if rejection_reason else 'v', rejection_reason)
#....
except Exception, e:
logger.error('Error while processing email', exc_info=True )
return None
else:
return True
How how I probe this issue?
It appears as if the error occurred before entering save_email(). With no commit() or rollback() directive in process_msg() a TransactionError is raised.
You could try to circle the error with a debugger:
def process_msg(m):
try:
import pdb # import python debugger
pdb.set_trace() # begin debugging
#....
save_email(u, l, d, t,
[…]
You can find information on how to use the debugger on the Python website.