Why a call to a celery task block forever? - django

I follow the tutorial at http://docs.celeryproject.org/en/master/django/first-steps-with-django.html
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cobros.settings')
app = Celery('cobros')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
# Settings
CELERY_APP="cobros"
CELERY_BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = TIME_ZONE
and call my first job:
#task
def add(x, y):
logger.info("Sum ", x, y)
return x + y
def syncClientAsync():
baseDir = getBaseDir(str(cobrador.id))
print("Delay...")
return add.delay(4, 4) #No work
return add.delay(4, 4).get(timeout=1) #No work
When I run the code python get stuck/blocked forever. Redis and Celery are running and not report any error or problem.

Related

Celery beat sends scheduled task only once and then doesn't send task when restarted

I am using Celery beat to perform a task that is supposed to be executed at on specific time. I was trying to excute it now by changing the time just to see if it works correctly. What I have noticed is it sends the task correctly when I run a fresh command that is celery -A jgs beat -l INFO but then suppose I change the time in the schedule section from two minutes or three minutes from now and then again run the above command, beat does not send the task. Then I noticed something strange. If I go to the admin area and delete all the other old tasks that were created in the crontab table, and then run the command again it sends the task again to the worker.
The tasks are being traced by the worker correctly and also the celery worker is working correctly. Below are the codes that I wrote to perform the task.
celery.py
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from django.conf import settings
from celery.schedules import crontab
from django.utils import timezone
from datetime import timezone
# Set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'jgs.settings')
app = Celery('jgs')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.conf.enable_utc = False
app.conf.update(timezone = 'Asia/Kolkata')
# app.conf.update(BROKER_URL=os.environ['REDIS_URL'],
# CELERY_RESULT_BACKEND=os.environ['REDIS_URL'])
app.config_from_object('django.conf:settings', namespace='CELERY')
# Celery beat settings
app.conf.beat_schedule = {
'send-expiry-email-everyday': {
'task': 'control.tasks.send_expiry_mail',
'schedule': crontab(hour=1, minute=5),
}
}
# Load task modules from all registered Django apps.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print(f'Request: {self.request!r}')
control/tasks.py
from celery import shared_task
from django.core.mail import message, send_mail, EmailMessage
from django.conf import settings
from django.template.loader import render_to_string
from datetime import datetime, timedelta
from account.models import CustomUser
from home.models import Contract
#shared_task
def send_expiry_mail():
template = render_to_string('expiry_email.html')
email = EmailMessage(
'Registration Successfull', #subject
template, # body
settings.EMAIL_HOST_USER,
['emaiid#gmail.com'], # sender email
)
email.fail_silently = False
email.content_subtype = 'html' # WITHOUT THIS THE HTML WILL GET RENDERED AS PLAIN TEXT
email.send()
return "Done"
settings.py
############# CELERY SETTINGS #######################
CELERY_BROKER_URL = 'redis://127.0.0.1:6379'
# CELERY_BROKER_URL = os.environ['REDIS_URL']
CELERY_ACCEPT_CONTENT =['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Asia/Kolkata'
CELERY_RESULT_BACKEND = 'django-db'
# CELERY BEAT CONFIGURATIONS
CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
commands that I am using
for worker
celery -A jgs.celery worker --pool=solo -l info
for beat
celery -A jgs beat -l INFO
Please correct me where I going wrong or what I am writing wrong, I completely in beginer phase in this async part.
I am really sorry if my sentences were confusing above.

Celery doesn't return or fail when calling apply_async. Works with celery_beat

I have a problem with calling celery tasks with apply_async.
I have in settings.py:
CELERY_BROKER_TRANSPORT_OPTIONS = {'confirm_publish': True}
CELERY_BROKER_URL = env('RABBITMQ_URL')
print(CELERY_BROKER_URL) //pyamqp://un:pw#app-rabbitmq:5672
CELERY_TASK_QUEUES = (
Queue('default', Exchange('default', type='direct'), routing_key='default'),
Queue('email', Exchange('email', type='direct'), routing_key='email'),
)
CELERY_TASK_ROUTES = {
'core.services.CoreSendEmailTaskService.*': {
'exchange': 'email',
'routing_key': 'email'
},
}
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_DEFAULT_QUEUE = 'default'
CELERY_TASK_ACKS_LATE = True
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_EXPIRES = 600
CELERY_RESULT_SERIALIZER = 'json'
CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND_URL') //redis://app-redis:6379/3
CELERY_RESULT_PERSISTENT = False
CELERY_WORKER_TASK_TIME_LIMIT = 65
CELERY_WORKER_TASK_SOFT_TIME_LIMIT = 60
CELERY_WORKER_HIJACK_ROOT_LOGGER = False
In project_config/celery_config/tasks/__init__.py:
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app
__all__ = ('app',)
and in project_config/celery_config/tasks/celery.py:
class CeleryApp(celery.Celery):
def on_configure(self):
sentry_dns = os.environ.get('DJANGO_SENTRY_DNS', None)
if sentry_dns and os.environ.get('ENVIRONMENT', 'local') == 'production':
client = raven.Client(
sentry_dns
)
register_logger_signal(client)
register_signal(client)
app = CeleryApp('tasks')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
class CeleryTasksConfig(AppConfig):
name = 'project_config.celery_config.tasks'
verbose_name = 'Celery Tasks'
def ready(self):
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
The weird thing is that the task executes on staging and production but locally it doesn't. Also locally when scheduled with beat the task runs normally.
The worker is locally started with command:
celery -A project_config.celery_config.tasks worker -O fair --loglevel=DEBUG --maxtasksperchild=1000 --queues=default,email -P prefork

celery task status showing pending

I am working with celery and i am getting tasks status is pending, may be it is implementation problem. please check my code.
I am trying to save task info like id, name, status in my mongodb database, for this i am using a function which my task will call to save data in mongodb.
Am i getting my task pending because my function call is happening before return statement of task?
settings.py
CELERY_BROKER_URL = 'mongodb://localhost:27017/jobs'
CELERY_RESULT_BACKEND = "mongodb"
CELERY_IGNORE_RESULT = False
CELERY_TRACK_STARTED = True
CELERY_MONGODB_BACKEND_SETTINGS = {
"host": "127.0.0.1",
"port": 27017,
"database": "jobs",
"taskmeta_collection": "my_taskmeta_collection",
}
CELERY_BEAT_SCHEDULE = {
'add-every-minute-contrab': {
'task': 'username_length_periodically',
'schedule': crontab(minute='*/1'),
#'args' : (2,3),
},
}
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = TIME_ZONE
celery.py
from __future__ import absolute_import, unicode_literals
import os, logging
from celery import Celery
from celery.schedules import crontab
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'RestUserAPI.settings')
app = Celery('UserAPI')
# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
tasks.py
from __future__ import absolute_import, unicode_literals
from celery import task, current_task, result
from django.conf import settings
import datetime
from .models import TaskMetaData
#task(name='username_length_periodically', bind=True)
def get_username_length_periodically(self):
last_run = datetime.datetime.now()
dict = {'name':self.name,
'id':self.request.id,
'status':self.AsyncResult(self.request.id).state,
'last_run': last_run,
}
store_metadata(dict)
return dict
def store_metadata(dict):
metadata = TaskMetaData()
metadata.task_id = dict['id']
metadata.task_name = dict['name']
metadata.task_status = dict['status']
metadata.task_last_run = dict['last_run']
metadata.save()
I think this is just a plain old logic error. If you take a look at your call to check the status of the task using AsyncResult:
'status':self.AsyncResult(self.request.id).state,
You'll notice that you are checking the status of the task, while the task is running. That means that the task will always show state PENDING (unless you have track_task_started set) when you check the task because you are always checking the status of the task from inside the task and then never go back and update the status!
In order to update the status of the task, you should kick off a separate monitoring task that periodically checks the status of the task and records it to the database until the tasks is finished or errors out. e.g.,
#app.task(name='monitor')
def monitor(task_id):
result = AsyncResult(task_id)
if result.state in celery.results.READY_STATES:
# update metadata table for the task_id
...
else:
monitor.apply_async(kwargs={ 'task_id': task_id }, countdown=60)

kombu utils objects py line 42 in __ get __ return obj __ dict __[ self __ name __] keyerror data

settings.py
CELERY_BROKER_URL = 'redis://localhost:6379'
CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Asia/Makassar'
# Other Celery settings
CELERY_BEAT_SCHEDULE = {
'add_task': {
'add_task': 'data_loader.tasks.add_task',
'schedule': crontab(minute=55, hour=13),
}
# 'task-number-two': {
# 'task': 'data_loader.tasks.demo',
# 'schedule': crontab(minute=2, hour='18'),
# }
}
celery.py
..............
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'resolution_validator.settings')
app = Celery('resolution_validator')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
tasks.py
............
from __future__ import absolute_import, unicode_literals
from celery.task import task
#task()
def add_task():
print("hello world")
a = 10
return True
init.py
...............
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ['celery_app']
I am also getting the following error
"RecursionError: maximum recursion depth exceeded" with the KeyError.
I am using Django==2.0,celery==4.2.0,python==3.5.2
Not able to get the solution .
There is a problem with your tasks.py. You should use #shared_task decorator instead of #task decorator
from __future__ import absolute_import, unicode_literals
from celery import shared_task
#shared_task
def add_task():
print("hello world")
a = 10
return True

Django - Celery with RabbitMQ: task always remains in PENDING

I have to use Celery 4.0.2 with RabbitMQ 3.6.10 to handle an async task. Then, I have followed this tutorial: https://www.codementor.io/uditagarwal/asynchronous-tasks-using-celery-with-django-du1087f5k
However, I have a slight problem with my task because it's impossible to have a result. My task always remains in "PENDING" state.
My question is what i have to do to get a result ?
Thank you in advance for your answer.
Here my code:
Here my __init_.py:
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ['celery_app']
Here a part of my setting.py:
BROKER_URL = 'amqp://guest:guest#localhost//'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
Here my celery.py:
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings')
app = Celery('mysite',
backend='amqp',
broker='amqp://guest#localhost//')
# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
And my tasks.py
# Create your tasks here
from __future__ import absolute_import, unicode_literals
from celery import shared_task
#shared_task
def add(x, y):
test = "ok"
current_task.update_state(state='PROGRESS',
meta={'test': ok})
return x + y
And here my Django Shell:
>>> from blog.tasks import *
>>> job = add.delay(2,3)
>>> job.state
'PENDING'
>>> job.result
>>>
With a picture of my RabbitMQ interface:
You need to start a worker that will process the tasks you add to queue. From your virtualenv run:
celery worker -A blog -l info