How to use Python threadpool and celery in combination? - flask

Question

Originally I have a small script doing some IO bound jobs using native Python threadpool, e.g.
from multiprocessing.dummy import Pool as ThreadPool
def f():
# some IO bound action here.
pass
def pooled_f():
pool = ThreadPool(4)
args = [1, 2, 3, 4]
results = []
for i in pool.imap_unordered(f, args):
results.append(i)
pool.close()
pool.join()
return results
Now I'm using Flask + Celery + Redis to allow user to submit different jobs to the worker and do it effeciently: the user submits a file containg some data, and celery would do some work, and showes the user progess. Maybe it's overkill for such a small app, but it should work anyway.
#celery.task(bind=True)
def sillywork(self, filename):
filesize = os.path.getsize(filename)
with open(filename, 'rb') as f:
chunked = 0
for i, line in enumerate(f):
message = line.strip()
chunked += len(message)
# print line
self.update_state(
state='PROGRESS',
meta={
'current': chunked,
'total': filesize,
'status': message
})
time.sleep(0.01)
return {
'current': filesize,
'total': filesize,
'status': 'OK',
'result': 1
}
#app.route('/jobs', methods=['POST', 'GET'])
def jobs():
form = Form()
if request.method == 'POST':
filename = form.data.filename
# file contains the data that user submitted
file = request.files['file']
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
# How should I add the celery job here?
sillywork.apply_async(args=[filename])
return redirect(url_for('index', task_id=task.id))
I followed the example https://blog.miguelgrinberg.com/post/using-celery-with-flask , somehow I couldn't find a way to make celery run the job I needed.