I am using the celery with python 3.5. I am using chain to perform serialize task and I only want to process single task at a time.
The follow piece of code works fine:
#shared_task(name='dummy1')
def dummy1(val):
logger.info('received: ' + val)
with RedisLock('testing.pid',60*60):
try:
logger.info("Lock acquired by " + val)
res = chain(dummy2.s(val), dummy3.s(val)).delay()
logger.info("Lock released")
except Exception as e:
logger.exception(str(e))
#shared_task(name='dummy2')
def dummy2(val):
logger.info('Dummy2 : ' + val)
time.sleep(3)
logger.info('Dummy2 complete')
return None
#shared_task(name='dummy3')
def dummy3(prev, val):
logger.info('Dummy3 : ' + val)
time.sleep(3)
logger.info('Dummy3 complete')
RedisLock is a distributed cache based lock for running a single task at a time.
However, when i do:
with RedisLock('testing.pid',60*60):
try:
logger.info("Lock acquired by " + val)
res = chain(dummy2.s(val), dummy3.s(val)).delay()
res.get() # blocking the process here
logger.info("Lock released")
the chain does not work as expected and gets crashed very often.
Related
I'm trying to use Paho MQTT Client and Multiprocessing to send temperature with defined interval. However, publish command is not working inside class. I've checked self.mqtt_client inside scheduler it has the Client object.
Is there anyone that can address problem for me?
Everything inside class is working except Scheduler.
def scheduler(self, topic, interval):
if interval != 0:
while True:
if topic == "temp":
print("Temperature published " + interval) #It's working.
self.mqtt_client.publish(topic, interval , 0 , False) #There is no error/output about this line
time.sleep(int(interval))
else:
pass
Class:
class Switcher:
config = None
mqtt_client = None
mqtt_connected = False
switches = {}
stages = {}
def __init__(self, config):
self.config = config
for switch_cfg in self.config['switches']:
self.switches[switch_cfg['topic_set']] = Switch(int(switch_cfg['gpio']), switch_cfg['topic_status'], switch_cfg['initial'])
def scheduler(self, topic, interval):
if interval != 0:
while True:
if topic == "temp":
print("Temperature published " + interval) #It's working.
self.mqtt_client.publish(topic, interval , 0 , False) #There is no error/output about this line
time.sleep(int(interval))
else:
pass
def mqtt_connect(self):
if self.mqtt_broker_reachable():
self.verbose('Connecting to ' + self.config['mqtt_host'] + ':' + self.config['mqtt_port'])
self.mqtt_client = mqtt.Client(self.config['mqtt_client_id'])
if 'mqtt_user' in self.config and 'mqtt_password' in self.config:
self.mqtt_client.username_pw_set(self.config['mqtt_user'], self.config['mqtt_password'])
self.mqtt_client.on_connect = self.mqtt_on_connect
self.mqtt_client.on_disconnect = self.mqtt_on_disconnect
self.mqtt_client.on_message = self.mqtt_on_message
try:
self.mqtt_client.connect(self.config['mqtt_host'], int(self.config['mqtt_port']), 10)
for switch_cfg in self.config['switches']:
self.mqtt_client.subscribe(switch_cfg['topic_set'], 0)
self.mqtt_client.loop_forever()
except:
self.error(traceback.format_exc())
self.mqtt_client = None
else:
self.error(self.config['mqtt_host'] + ':' + self.config['mqtt_port'] + ' not reachable!')
def mqtt_on_connect(self, mqtt_client, userdata, flags, rc):
self.mqtt_connected = True
for switch_ios in self.config['switches']:
self.mqtt_client.publish(self.config['station_status'], "available", 0, False)
self.mqtt_client.publish(switch_ios['topic_status'], self.switches[switch_ios['topic_set']].get_state(), 0, False)
temp_interval = 1
temp_process = multiprocessing.Process(target=self.scheduler, args=("temp",str(temp_interval),))
temp_process.start()
self.verbose('...mqtt_connected!')
def mqtt_broker_reachable(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5)
try:
s.connect((self.config['mqtt_host'], int(self.config['mqtt_port'])))
s.close()
return True
except socket.error:
return False
def start(self):
self.mqtt_connect()
You mqtt_connect function will never return.
self.mqtt_client.loop_forever() Will block until self.mqtt_client.disconnect() is called.
You should probably be using self.mqtt_client.loop_start() which will run the client loop on it's own thread in the background. You can call self.mqtt_client.loop_stop() when you want to shut the client down.
My final code is looks like the following:
#client_route.route('/client/cat-<dir_code>/subscribe/<string:user_id>', methods=['POST'])
#only_client
#is_active_client
# #cache.cached(timeout=60)
def subscribe_abonent(user_id):
search_form = Search()
client_id = Client.query.filter_by(tele=session.get('client_phone') ,name=session.get('client_logged_in'), family=session.get('client_family')).first_or_404()
master = Abonent.query.filter_by(public_id=user_id, slug_direction=g.get('current_directory')).first_or_404()
client = Client.query.filter_by(id=client_id.id).first_or_404()
if session.get('client_is_not_subscriber'):
session.pop('client_is_not_subscriber')
session['client_is_subscriber'] = client.name + client.family + str(client.id)
socketio.emit('show_calendar', namespace='/masterCalendar-{}'.format(master.slug, master.public_id.replace('-','')))
if master.email is not None:
if master.subscribers_notifications:
date_now = datetime.now()
send_email(master.email, 'You got a new subscriber','client/newsletter/new_subscriber',master=master.name + ' ' + master.family, client=client.name + " " + client.family, sent=date_now)
#celery.task
def send_email(to, subject, template, **kwargs):
with current_app.app_context():
msg = Message(current_app.config['UBOOK_MAIL_SUBJECT_PREFIX'] + ' ' + subject,
sender=current_app.config['UBOOK_MAIL_SENDER'], recipients=[to])
msg.body = render_template(template + '.txt', **kwargs)
msg.html = render_template(template + '.html', **kwargs)
mail.send(msg)
Now everything is working fine, but celery not recognizing the task, the task runs inside the terminal just like your are sending email using Thread !!!
new_subscriber.txt:
Dear Vladimir, Here was {{master}}
You got a new subscriber:
Клиент: Sasha , Here was {{client}}
Получено: 2017-11-23 12:47:23 , Here was {{sent}}
Another thing that i forgot to mention that , if i removed the master, client and sent arguments and used delay() the code get freezed !!
I am trying to send a file across the network using Twisted with the LineReceiver protocol. The issue I am seeing is that when I read a binary file and try to send the chunks they simply don't send.
I am reading the file using:
import json
import time
import threading
from twisted.internet import reactor, threads
from twisted.protocols.basic import LineReceiver
from twisted.internet import protocol
MaximumMsgSize = 15500
trySend = True
connectionToServer = None
class ClientInterfaceFactory(protocol.Factory):
def buildProtocol(self, addr):
return WoosterInterfaceProtocol(self._msgProcessor, self._logger)
class ClientInterfaceProtocol(LineReceiver):
def connectionMade(self):
connectionToServer = self
def _DecodeMessage(self, rawMsg):
header, body = json.loads(rawMsg)
return (header, json.loads(body))
def ProcessIncomingMsg(self, rawMsg, connObject):
# Decode raw message.
decodedMsg = self._DecodeMessage(rawMsg)
self.ProccessTransmitJobToNode(decodedMsg, connObject)
def _BuildMessage(self, id, msgBody = {}):
msgs = []
fullMsgBody = json.dumps(msgBody)
msgBodyLength = len(fullMsgBody)
totalParts = 1 if msgBodyLength <= MaximumMsgSize else \
int(math.ceil(msgBodyLength / MaximumMsgSize))
startPoint = 0
msgBodyPos = 0
for partNo in range(totalParts):
msgBodyPos = (partNo + 1) * MaximumMsgSize
header = {'ID' : id, 'MsgParts' : totalParts,
'MsgPart' : partNo }
msg = (header, fullMsgBody[startPoint:msgBodyPos])
jsonMsg = json.dumps(msg)
msgs.append(jsonMsg)
startPoint = msgBodyPos
return (msgs, '')
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
for filename in exportedFiles:
with open (filename, "rb") as exportFileHandle:
data = exportFileHandle.read().encode('base64')
params = {
ExportFileToMaster_Tag.TaskID : taskID,
ExportFileToMaster_Tag.FileContents : data,
ExportFileToMaster_Tag.Filename : filename
}
msgs, _ = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
connection.sendLine(m)
def lineReceived(self, data):
threads.deferToThread(self.ProcessIncomingMsg, data, self)
def ConnectFailed(reason):
print 'Connection failed..'
reactor.callLater(20, reactor.callFromThread, ConnectToServer)
def ConnectToServer():
print 'Connecting...'
from twisted.internet.endpoints import TCP4ClientEndpoint
endpoint = TCP4ClientEndpoint(reactor, 'localhost', 8181)
deferItem = endpoint.connect(factory)
deferItem.addErrback(ConnectFailed)
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
reactor.callFromThread(ConnectToServer)
factory = ClientInterfaceFactory()
protocol = ClientInterfaceProtocol()
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
Is there something I am doing wrong?file is sent, it's when the write is multi part or there are more than one file it struggles.
If a single write occurs then the m
Note: I have updated the question with a crude piece of sample code in the hope it makes sense.
_BuildMessage returns a two-tuple: (msgs, '').
Your network code iterates over this:
msgs = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
So your network code first tries to send a list of json encoded data and then tries to send the empty string. It most likely raises an exception because you cannot send a list of anything using sendLine. If you aren't seeing the exception, you've forgotten to enable logging. You should always enable logging so you can see any exceptions that occur.
Also, you're using time.sleep and you shouldn't do this in a Twisted-based program. If you're doing this to try to avoid overloading the receiver, you should use TCP's native backpressure instead by registering a producer which can receive pause and resume notifications. Regardless, time.sleep (and your loop over all the data) will block the entire reactor thread and prevent any progress from being made. The consequence is that most of the data will be buffered locally before being sent.
Also, your code calls LineReceiver.sendLine from a non-reactor thread. This has undefined results but you can probably count on it to not work.
This loop runs in the main thread:
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
while the reactor runs in another thread:
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
ProcessTransmitJobToNode simply calls self.sendLine:
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
You should probably remove the use of threading entirely from the application. Time-based events are better managed using reactor.callLater (your main-thread loop effectively generates a call to ProcessTransmitJobToNode once hundred times a second (modulo effects of the trySend flag)).
You may also want to take a look at https://github.com/twisted/tubes as a better way to manage large amounts of data with Twisted.
I am using a MapReduce code, and I met a problem: the code doesn't respond after the map (takes 1 hr) finished. I digged into the code and find this function doesn't respond:
def wait(self, running, tag): """Test if any worker has finished its job. If so, decrease its key and make it available """
atimer = Timer('Wait')
inittime = time()
status = MPI.Status()
while time() - inittime < self.config['jobwait']:
if world.Iprobe(source=MPI.ANY_SOURCE,tag=tag,status=status):
jobf = world.recv(source=status.source, tag=tag)
idx = 0
for ii, worker in enumerate(self.workers):
if worker.id == status.source: idx = ii; break
if self.config['verbosity'] >= 8:
print('Freeing worker '+str(self.workers[idx].id))
worker = self.workers[idx]
# faulty worker's job has already been cleaned
if not worker.isFaulty():
del running[jobf]
else:
self.nActive += 1
worker.setFree()
heapq._siftup(self.workers, idx)
This line doesn't respond :
if world.Iprobe(source=MPI.ANY_SOURCE,tag=tag,status=status):
I wonder if there is a time out for Iprobe() in mpi4py and how to set the timeout time for it? Is there a alternative for Iprobe() that has the same role here?
Here is the previous function that send the message via .send()
def execTask(self, task):
"""Wrapper function calling mapping/reducing/finalizing phase tasks,
dispatch tasks to workers until all finished and collect feedback.
Faulty workers are removed from active duty work list.
"""
atimer = Timer(task)
print( 'Entering {0:s} phase...'.format(task) )
taskDict = { 'Map':(self.mapIn, MAP_START, MAP_FINISH), \
'Init':(self.mapIn, INIT_START, MAP_FINISH), \
'Reduce':(self.reduceIn, REDUCE_START, REDUCE_FINISH) }
# line up jobs and workers into priority queues
jobs = taskDict[task][0][:]
heapq.heapify(jobs); running = {}
heapq.heapify(self.workers)
while (jobs or running) and self.nActive > 0:
# dispatch all jobs to all free workers
while jobs and self.workers[0].isFree():
job = heapq.heappop(jobs)
worker = heapq.heappop(self.workers)
world.send(job, dest=worker.id, tag=taskDict[task][1])
print('hi')
print job
worker.setBusy(); heapq.heappush(self.workers, worker)
running[job] = (time(), worker)
if self.config['verbosity'] >= 6:
print('Dispatching file '+os.path.basename(job)+' to worker '+str(worker.id))
# if no more free workers, break
if not self.workers[0].isFree(): break
# wait for finishing workers as well as do cleaning
self.wait(running, taskDict[task][2])
# print running
self.clean(running, jobs)
print( '{0:s} phase completed'.format(task) )
The whole code can be seen here:
https://drive.google.com/file/d/0B36fJi35SPIedWdjbW5NdzlCeTg/view?usp=sharing
I am making a simple math test for my friend's class. The students will only have 45 seconds to solve each answer. Is there a way to make a timer that will count at the same time as the rest of the code runs and when it reaches 45 stops?
The test looks like this:
test = raw_input("How much is 62x5-23?")
if test == '287':
print "Well done!"
Here's some code I used once (lifted off some page on the web which is now in the hands of a domain squatter, so no credit where credit is due, sadly):
import signal
class TimeoutException(Exception):
pass
def timeout(timeout_time, default = None):
def timeout_function(f):
def f2(*args, **kwargs):
def timeout_handler(signum, frame):
raise TimeoutException()
old_handler = signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(timeout_time) # triger alarm in timeout_time seconds
try:
retval = f(*args, **kwargs)
except TimeoutException, e:
if default == None:
raise e
return default
finally:
signal.signal(signal.SIGALRM, old_handler)
signal.alarm(0)
return retval
return f2
return timeout_function
# use like this:
#timeout(45)
def run():
test = raw_input("How much is 62x5-23? ")
if test == '287':
print "Well done!"
# alternatively, pass a value that will be returned when the timeout is reached:
#timeout(45, False)
def run2():
test = raw_input("How much is 62x5-23? ")
if test == '287':
print "Well done!"
if __name__ == '__main__':
try:
run()
except TimeoutException:
print "\nSorry, you took too long."
# alternative call:
if run2() == False:
print "\nSorry, you took too long."
EDIT: probably works on Unix-type OS'es only.