I want to run this function everyday midnight to check expiry_date_notification. what can I do? I'm new to django and python.
def check_expiry_date(request):
products = Product.objects.all()
for product in products:
product_id = product.id
expiry_date = product.expiry_date
notification_days = product.notification_days
check_date = int((expiry_date - datetime.datetime.today()).days)
if notification_days <= check_date:
notification = Notification(product_id=product_id)
notification.save()
As others have said, Celery can schedule tasks to execute at a specific time.
from celery.schedules import crontab
from celery.task import periodic_task
#periodic_task(run_every=crontab(hour=7, minute=30, day_of_week="mon"))
def every_monday_morning():
print("This is run every Monday morning at 7:30")
Install via pip install django-celery
You can either write a custom management command and schedule its execution using cron, or you can use celery.
Have a look at:
Celery - Distributed Task Queue
Related
I have a Group model:
class Group(models.Model):
leader = models.ForeignKey(User, on_delete=models.CASCADE)
name = models.CharField(max_length=55)
description = models.TextField()
joined = models.ManyToManyField(User, blank=True)
start_time = models.TimeField(null=True)
end_time = models.TimeField(null=True)
email_list = ArrayField(
models.CharField(max_length=255, blank=True),
blank=True,
default=list,
)
and I want to send an email to all Users who have joined a particular Group 30 minutes before the start_time. For example: if a Group has a start_time of 1:00 PM, I want to send an email to all the joined Users at 12:30 PM, letting them know the group will be meeting soon.
I currently have a bunch of celery tasks that run without error, but they are all called within views by the User (creating, updating, joining, leaving, and deleting groups will trigger a celery task to send an email notification to the User).
The scheduled email I am trying to accomplish here will be a periodic task, I assume, and not in the control of the User. However, it isn't like other periodic tasks I've seen because the time it relies on is based on the start_time of a specific Group.
#Brian in the comments pointed out that it can be a regular celery task that is called by the periodic task every minute. Here's my celery task:
from celery import shared_task
from celery.utils.log import get_task_logger
from django.core.mail import send_mail
from my_chaburah.settings import NOTIFICATION_EMAIL
from django.template.loader import render_to_string
#shared_task(name='start_group_notification_task')
def start_group_notification_task(recipients):
logger.info('sent email to whole group that group is starting')
for recipient in recipients:
send_mail (
'group starting',
'group starting',
NOTIFICATION_EMAIL,
[recipient],
fail_silently=False
)
I'm still not sure exactly how to call this task using a periodic task or how to query my groups and find when groups start_time == now + 30mins. I've read the docs, but I'm new to celery and celery beat and a bit confused by how to move forward.
I'm also not sure where exactly to call the task.
my myapp/celery.py file:
import os
from celery import Celery
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'my_group.settings')
app = Celery('my_group')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
#app.task(bind=True, ignore_result=True)
def debug_task(self):
print(f'Request: {self.request!r}')
my group/tasks.py file:
from celery import shared_task
from celery.utils.log import get_task_logger
from django.core.mail import send_mail
from my_chaburah.settings import NOTIFICATION_EMAIL
from django.template.loader import render_to_string
logger = get_task_logger(__name__)
I have a bunch of tasks that I didn't include, but I'm assuming any task regarding my Group model would go here. Still not sure though.
I'd also like to add the ability for the leader of the Group to be able to set the amount time prior to start_time where the email will be sent. For example: 10, mins, 30 mins, 1hr before meeting, but that's more to do with the model.
You can follow the steps here to configure celery for periodic tasks.
In line with that you can do something roughly similar to this:
import datetime
from celery import Celery
from myapp.models import Group
app = Celery()
#app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
# Setup and call send_reminders() every 60 seconds.
sender.add_periodic_task(60.0, send_reminders, name='check reminders to be sent every minute')
#app.task
def send_reminders():
# Celery task that gets all groups that needs reminders to be sent 30 minutes from now
thirty_minutes_from_now = datetime.datetime.now() + datetime.timedelta(minutes=30)
groups = Group.objects.filter(
start_time__hour=thirty_minutes_from_now.hour,
start_time__minute=thirty_minutes_from_now.minute
).prefetch_related("joined")
for group in groups:
for member in group.joined.all():
send_email_task.delay(member.email)
#app.task
def send_email_task(recipient):
# Celery task to send emails
send_mail(
'group starting',
'group starting',
NOTIFICATION_EMAIL,
[recipient],
fail_silently=False
)
Disclaimer: This is not tested and optimised ;)
Why periodic tasks doesn't work?
There has the bug in Celery <= 5.2.7(stable-version).
It let the periodic tasks doesn't work.
I'm fixed it in this PR, you can edit your Celery source code like this PR, or try the Celery dev-version.
Soluction 1
# your_app/task.py
#app.on_on_after_finalize.connect
def setup_periodic_tasks(sender, **kwargs):
for group in Group.object.all():
notification_time = group.start_time - timedelta(minutes=30)
sender.add_periodic_task(clocked(notification_time),
start_group_notification_task,
kwargs={'recipients':group.recipients}
name='send mail when group start time')
# You need to connect the `Group.post_save` and `Group.post_delete` signal here,
# to setup/revoke your periodic tasks when `Group` changed.
You can use the clocked to be your custom scheduler class, I quoted it from django-celery-beat.
Soluction 2
Maybe you can use the django-celery-beat, and create m2m related to your Group and PeriodicTask.
It looks easier.
Disclaimer: This is not tested and optimised too.
I was able to figure out how to run the task based on start_time but am concerned about issues with runtime.
I added this to my celery.py file:
app.conf.beat_schedule = {
'start_group_notification': {
'task': 'start_group_notification_task',
'schedule': crontab(),
}
}
Which runs the task every minute. The task then checks to see if a Group has a start time within 30 minutes. In group/tasks.py
#shared_task(name='start_group_notification_task')
def start_group_notification_task():
logger.info('sent email to whole group that group is starting')
thirty_minutes_from_now = datetime.datetime.now() + datetime.timedelta(minutes=30)
groups = Group.objects.filter(
start_time__hour=thirty_minutes_from_now.hour,
start_time__minute=thirty_minutes_from_now.minute
).prefetch_related("joined")
for group in groups:
for email in group.email_list:
send_mail (
'group starting in 30 minutes',
group.name,
NOTIFICATION_EMAIL,
[email],
fail_silently=False
)
Now, this works, but I'm concerned about having nested for loops. Is there maybe a better way to do this to have runtime be as little as possible? Or are celery tasks executed fast enough and easy enough that it's not an issue.
As #Brian mentioned this doesn't take into account Users joining within the 30 minute period before a Group starts. The fix to this is for me to see if User joins group within that 30 min period and call a different task to tell them the group is starting soon.
EDIT:
If a User joins a Group within the 30 minute window, I added this variable and conditional:
time_left = int(chaburah.start_time.strftime("%H%M")) - int(current_date.strftime("%H%M"))
if time_left <= 30 and time_left >= 0:
celery_task.delay()
That works if a User joins within the 30, but if the Group has already started I have to implement a new task to let the User know the Group has started.
I trying to run tasks with celery-beat. When I launch it by minute or hour schedule, tasks will start correctly, but if I trying to run daily task, it display in django admin panel, but not run in time.
It must to work in the following way: regular django code starts a 'start_primaries' task in Party class:
def setup_task(self):
schedule, created = IntervalSchedule.objects.get_or_create(every=7, period=IntervalSchedule.DAYS)
self.task = PeriodicTask.objects.create(
name=self.title + ', id ' + str(self.pk),
task='start_primaries',
interval=schedule,
args=json.dumps([self.id]),
start_time=timezone.now()
)
self.save()
Is it possible that there are some settings that limit the duration of the task's life? At the moment I have the following among the Django settings:
CELERY_TASK_TRACK_STARTED = True
CELERY_TASK_TIME_LIMIT = 950400
CELERY_BROKER_URL = 'redis://redis:6379/0'
CELERY_RESULT_BACKEND = 'django-db'
CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
CELERY_TASK_RESULT_EXPIRES = None
CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 950400}
Finally,I found the anwser in github.
First, You should set 'last_run_at' to start_time - interval, so that beat will check when it was last "executed", so the next time it executes will be at start_time.
Second, update your start_time setting, and let the scheduler get the updated info.
Good luck.
https://github.com/celery/django-celery-beat/issues/259
I feel a bit stupid for asking, but it doesn't appear to be in the documentation for RQ. I have a 'failed' queue with thousands of items in it and I want to clear it using the Django admin interface. The admin interface lists them and allows me to delete and re-queue them individually but I can't believe that I have to dive into the django shell to do it in bulk.
What have I missed?
The Queue class has an empty() method that can be accessed like:
import django_rq
q = django_rq.get_failed_queue()
q.empty()
However, in my tests, that only cleared the failed list key in Redis, not the job keys itself. So your thousands of jobs would still occupy Redis memory. To prevent that from happening, you must remove the jobs individually:
import django_rq
q = django_rq.get_failed_queue()
while True:
job = q.dequeue()
if not job:
break
job.delete() # Will delete key from Redis
As for having a button in the admin interface, you'd have to change django-rq/templates/django-rq/jobs.html template, who extends admin/base_site.html, and doesn't seem to give any room for customizing.
The redis-cli allows FLUSHDB, great for my local environment as I generate a bizzallion jobs.
With a working Django integration I will update. Just adding $0.02.
You can empty any queue by name using following code sample:
import django_rq
queue = "default"
q = django_rq.get_queue(queue)
q.empty()
or even have Django Command for that:
import django_rq
from django.core.management.base import BaseCommand
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument("-q", "--queue", type=str)
def handle(self, *args, **options):
q = django_rq.get_queue(options.get("queue"))
q.empty()
As #augusto-men method seems not to work anymore, here is another solution:
You can use the the raw connection to delete the failed jobs. Just iterate over rq:job keys and check the job status.
from django_rq import get_connection
from rq.job import Job
# delete failed jobs
con = get_connection('default')
for key in con.keys('rq:job:*'):
job_id = key.decode().replace('rq:job:', '')
job = Job.fetch(job_id, connection=con)
if job.get_status() == 'failed':
con.delete(key)
con.delete('rq:failed:default') # reset failed jobs registry
The other answers are outdated with the RQ updates implementing Registries.
Now, you need to do this to loop through and delete failed jobs. This would work any particular Registry as well.
import django_rq
from rq.registry import FailedJobRegistry
failed_registry = FailedJobRegistry('default', connection=django_rq.get_connection())
for job_id in failed_registry.get_job_ids():
try:
failed_registry.remove(job_id, delete_job=True)
except:
# failed jobs expire in the queue. There's a
# chance this will raise NoSuchJobError
pass
Source
You can empty a queue from the command line with:
rq empty [queue-name]
Running rq info will list all the queues.
I'm struggling with Django, Celery, djcelery & PeriodicTasks.
I've created a task to pull a report for Adsense to generate a live stat report. Here is my task:
import datetime
import httplib2
import logging
from apiclient.discovery import build
from celery.task import PeriodicTask
from django.contrib.auth.models import User
from oauth2client.django_orm import Storage
from .models import Credential, Revenue
logger = logging.getLogger(__name__)
class GetReportTask(PeriodicTask):
run_every = datetime.timedelta(minutes=2)
def run(self, *args, **kwargs):
scraper = Scraper()
scraper.get_report()
class Scraper(object):
TODAY = datetime.date.today()
YESTERDAY = TODAY - datetime.timedelta(days=1)
def get_report(self, start_date=YESTERDAY, end_date=TODAY):
logger.info('Scraping Adsense report from {0} to {1}.'.format(
start_date, end_date))
user = User.objects.get(pk=1)
storage = Storage(Credential, 'id', user, 'credential')
credential = storage.get()
if not credential is None and credential.invalid is False:
http = httplib2.Http()
http = credential.authorize(http)
service = build('adsense', 'v1.2', http=http)
reports = service.reports()
report = reports.generate(
startDate=start_date.strftime('%Y-%m-%d'),
endDate=end_date.strftime('%Y-%m-%d'),
dimension='DATE',
metric='EARNINGS',
)
data = report.execute()
for row in data['rows']:
date = row[0]
revenue = row[1]
try:
record = Revenue.objects.get(date=date)
except Revenue.DoesNotExist:
record = Revenue()
record.date = date
record.revenue = revenue
record.save()
else:
logger.error('Invalid Adsense Credentials')
I'm using Celery & RabbitMQ. Here are my settings:
# Celery/RabbitMQ
BROKER_HOST = "localhost"
BROKER_PORT = 5672
BROKER_USER = "myuser"
BROKER_PASSWORD = "****"
BROKER_VHOST = "myvhost"
CELERYD_CONCURRENCY = 1
CELERYD_NODES = "w1"
CELERY_RESULT_BACKEND = "amqp"
CELERY_TIMEZONE = 'America/Denver'
CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
import djcelery
djcelery.setup_loader()
On first glance everything seems to work, but after turning on the logger and watching it run I have found that it is running the task at least four times in a row - sometimes more. It also seems to be running every minute instead of every two minutes. I've tried changing the run_every to use a crontab but I get the same results.
I'm starting celerybeat using supervisor. Here is the command I use:
python manage.py celeryd -B -E -c 1
Any ideas as to why its not working as expected?
Oh, and one more thing, after the day changes, it continues to use the date range it first ran with. So as days progress it continues to get stats for the day the task started running - unless I run the task manually at some point then it changes to the date I last ran it manually. Can someone tell me why this happens?
Consider creating a separate queue with one worker process and fixed rate for this type of tasks and just add the tasks in this new queue instead of running them in directly from celerybeat. I hope that could help you to figure out what is wrong with your code, is it problem with celerybeat or your tasks are running longer than expected.
#task(queue='create_report', rate_limit='0.5/m')
def create_report():
scraper = Scraper()
scraper.get_report()
class GetReportTask(PeriodicTask):
run_every = datetime.timedelta(minutes=2)
def run(self, *args, **kwargs):
create_report.delay()
in settings.py
CELERY_ROUTES = {
'myapp.tasks.create_report': {'queue': 'create_report'},
}
start additional celery worker with that would handle tasks in your queue
celery worker -c 1 -Q create_report -n create_report.local
Problem 2. Your YESTERDAY and TODAY variables are set at class level, so within one thread they are set only once.
I'm running multiple simulations as tasks through celery (version 2.3.2) from django. The simulations get set up by another task:
In views.py:
result = setup_simulations.delay(parameters)
request.session['sim'] = result.task_id # Store main task id
In tasks.py:
#task(priority=1)
def setup_simulations(parameters):
task_ids = []
for i in range(number_of_simulations):
result = run_simulation.delay(other_parameters)
task_ids.append(result.task_id)
return task_ids
After the initial task (setup_simulations) has finished, I try to revoke the simulation tasks as follows:
main_task_id = request.session['sim']
main_result = AsyncResult(main_task_id)
# Revoke sub tasks
from celery.task.control import revoke
for sub_task_id in main_result.get():
sub_result = AsyncResult(sub_task_id); sub_result.revoke() # Does not work
# revoke(sub_task_id) # Does not work neither
When I look at the output from "python manage.py celeryd -l info", the tasks get executed as if nothing had happened. Any ideas somebody what could have gone wrong?
As you mention in the comment, revoke is a remote control command so it's only currently supported by the amqp and redis transports.
You can accomplish this yourself by storing a revoked flag in your database, e.g:
from celery import states
from celery import task
from celery.exceptions import Ignore
from myapp.models import RevokedTasks
#task
def foo():
if RevokedTasks.objects.filter(task_id=foo.request.id).count():
if not foo.ignore_result:
foo.update_state(state=states.REVOKED)
raise Ignore()
If your task is working on some model you could even store a flag in that.