Celery Tasks on Django Models - django

I'm trying to learn how to use celery to check a date every day on one of my models. One of my models holds an expiration date and a boolean field that says whether their insurance is expired or not.
The model is pretty big so I'm going to post a condensed version. I think I have two options. Either run a celery task on the model method or rewrite the function in my tasks.py. Then I need to use Celery beat to run the schedule to check daily.
I've got the function to work, but I'm passing in the model objects directly which I think is wrong.
I'm also having trouble on how to use the args in the celery beat scheduler inside of my celery.py.
I'm really close to getting this to work, but I think I'm going about executing a task in the wrong way. and I think executing a task on the model method might be the cleanest, I'm just not sure how to accomplish that.
models.py
class CarrierCompany(models.Model):
name = models.CharField(max_length=255, unique=True)
insurance_expiration = models.DateTimeField(null=True)
insurance_active = models.BooleanField()
def insurance_expiration_check(self):
if self.insurance_expiration > datetime.today().date():
self.insurance_active = True
self.save()
print("Insurance Active")
else:
self.insurance_active = False
self.save()
print("Insurance Inactive")
tasks.py
from __future__ import absolute_import, unicode_literals
from celery.decorators import task
from datetime import datetime, date
from django.utils import timezone
from .models import CarrierCompany
#task(name="insurance_expired")
def insurance_date():
carriers = CarrierCompany.objects.all()
for carrier in carriers:
date = datetime.now(timezone.utc)
if carrier.insurance_expiration > date:
carrier.insurance_active = True
carrier.save()
print("Insurance Active")
else:
carrier.insurance_active = False
carrier.save()
print("Insurance Inactive")
celery.py
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from celery.schedules import crontab
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')
app = Celery('POTRTMS')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
app.conf.beat_schedule = {
'check-insurance-daily': {
'task': 'insurance_expired',
'schedule': crontab(hour='8')
},
}
*** updated the beat schedule to reflect when I actually want to run it.

An example for how I might do it would be as follows. Also, instead of using traditional datetime, if you are including timezones in your Django App, you will probably want to use the timezone library instead found here.
models.py
class CarrierCompany(models.Model):
...
#property
def is_insurance_expired(self):
from django.utils import timezone
if self.insurance_expiration > timezone.datetime.today():
print("Insurance Active")
return True
else:
print("Insurance Active")
return False
tasks.py
def insurance_date():
carriers = CarrierCompany.objects.all()
for carrier in carriers:
if carrier.is_insurance_expired:
carrier.insurance_active = True
carrier.save()
else:
carrier.insurance_active = False
carrier.save()
There are other things you could do, like not update it if its False and make the default False, or vice versa in the True case. You could also just do all of that in the models function, though I personally like to keep logic a bit separate (just how I organize my stuff). Hopefully this helps get you past where you are stuck though.

Related

Getting error by setting a Dynamic Task Scheduling With Django-celery-beat

I'm setting up Dynamic cron based on the model creation to create another model object
error im getting is
Cannot assign "(<CrontabSchedule: 0 0 27 1 * (m/h/dM/MY/d) UTC>,
True)": "PeriodicTask.crontab" must be a "CrontabSchedule" instance.
this is my code where a periodic task will be created by creating this model
import json
from django.utils import timezone
import uuid
from django.db import models
from django_celery_beat.models import CrontabSchedule, PeriodicTask
# Create your models here.
class Rule(models.Model):
rule_id = models.UUIDField(
primary_key = False,
default = uuid.uuid4,
editable = True)
name = models.CharField('r=Rule Name',max_length=255,blank=True,null=True)
compliance_frequency = models.CharField(max_length=40,choices=(('QUARTERLY','QUARTERLY'),
('MONTHLY','MONTHLY'),('YEARLY','YEARLY')),default='YEARLY')
applicable_from = models.DateField(default=timezone.now)
applicable_to = models.DateField(null=True,blank=True)
task = models.OneToOneField(
PeriodicTask,
on_delete=models.CASCADE,
null=True,
blank=True
)
def setup_task(self):
self.task = PeriodicTask.objects.create(
name=self.name,
task='create_compliance',
crontab=self.crontab_schedule,
args=json.dumps([self.id,self.name,self.compliance_frequency]),
start_time= timezone.now(),
)
self.save()
#property
def crontab_schedule(self):
if self.compliance_frequency == "QUARTERLY":
return CrontabSchedule.objects.get_or_create(minute='0',hour='0',day_of_week='*',
day_of_month='27',month_of_year='3,6,9,12')
if self.compliance_frequency == "MONTHLY":
return CrontabSchedule.objects.get_or_create(minute='0',hour='0',day_of_week='*',
day_of_month='27',month_of_year='*')
if self.compliance_frequency == 'YEARLY':
return CrontabSchedule.objects.get_or_create(minute='0',hour='0',day_of_week='*',
day_of_month='27',month_of_year='1')
raise NotImplementedError(
'''Interval Schedule for {interval} is not added.'''.format(
interval=self.compliance_frequency.
value))
now using signal.py i'm triggering the setup_task to setup the task
from django.db.models.signals import post_save
from django.dispatch import receiver
from .models import Rule
#receiver(post_save,sender=Rule)
def create_periodic_task(sender, instance, created, **kwargs):
if created:
instance.setup_task()
instance.task.enabled = True
instance.task.save()
print('signal working')
then using trigger a task to create a model
from __future__ import absolute_import, unicode_literals
from celery.utils.log import get_task_logger
from celery import shared_task
from pytz import timezone
from test3.models import Compliance
logger = get_task_logger(__name__)
#shared_task
def create_compliance(id,name,compliance_frequency):
Compliance.objects.create(rule_id=id,name=name,compliance_frequency=compliance_frequency)
id = Compliance.object.get(id=id)
logger.info(id.id,'periodic task working bitch')

Using RabbitMQ with Celery in Django post_save

I am applying asynchronous task handling using Celery on my Django Project.
My project's logic:,
from frontend side, there is a table with rows each having an upload button. User clicks on it and a payload is sent to backend containing a url that contains a file.
File is recieved in django views. And saved into database, table Run. Immediately object is saved a post_save signal is triggered to run a celery task.
The task to be performed is, fetch a list of runs with specific status. For each run, perform a task of downloading the file.
I would like to perform this asynchronously in case there is more than one run. Keeping in mind user can click upload for more than one row from frontend.
I am setting up RabbitMQ as my broker. I have rabbitMQ installed and running. I have set the CELERY_BROKER_URL='amqp://localhost' too in settings.py. I am a little lost on what I should do next in my configurations, could I get some guidance. I think I need to configure celery worker on my tasks.
Below is my code so far :
views.py #view that saves to database
class RunsUploadView(APIView):
serializer_class = RunsURLSerializer
def post(self, request, *args, **kwargs):
crawler_name = self.request.data.get('crawler')
run_id = self.kwargs.get("run_id")
run_url = self.request.data.get("run_url")
run = Run()
run.name = f"{crawler_name}_{run_id}"
run.run = run_id
run.url = run_url
run.save()
return Response(model_to_dict(run))
models.py # run is saved to table Run then a post_save signal is triggered.
from django.db import models
class Run(models.Model):
UPLOAD_STATUS = (
("Pending", "pending"),
("Running", "running"),
("Success", "success"),
("Failed", "failed"),
)
name = models.CharField(max_length=100)
run = models.CharField(max_length=100, unique=True)
url = models.URLField(max_length=1000)
status = models.CharField(
max_length=50, choices=UPLOAD_STATUS, default="Pending")
started_at = models.DateTimeField(null=True)
done_at = models.DateTimeField(null=True)
signals.py #handling the post_save logic after save()
from django.db.models.signals import post_save
from django.dispatch import receiver
from main.models import Run
from main.tasks import DownloadRun
#receiver(post_save, sender=Run)
def download_file(sender, **kwargs):
pending_runs = Run.objects.filter(status='Pending') #all pending runs collected, I would need to handle the runs asynchronously.
for run in pending_runs:
run.status = "Started"
run.save()
DownloadRun(run)
Tasks.py #using a class because I am going to update with more functions.
class DownloadRun:
def __init__(self, run):
run_object = model_to_dict(run)
self.run_url = run_object["url"]
self.download_run()
def download_run(self, dest_folder="runs"):
""Run file is downloaded from url""
I figured out the way forward. I did not configure celery well.
tasks.py
from celery import shared_task #import shared_task decorator from celery
class DownloadRun:
def __init__(self, run):
run_object = model_to_dict(run)
self.run_url = run_object["url"]
self.download_run()
def download_run(self, dest_folder="runs"):
""Run file is downloaded from url""
#shared_task
def celery_task(run_id):
DownloadRun(run_id)
signals.py
from django.db.models.signals import post_save
from django.dispatch import receiver
from main.models import Run
from main.tasks import celery_task #import celery_task from tasks.py
#receiver(post_save, sender=Run)
def download_file(sender, **kwargs):
pending_runs = Run.objects.filter(status='Pending')
for run in pending_runs:
run.status = "Started"
run.save()
celery_task.delay(run.run) #call celery delay() to invoke the task (pass the unique key as parameter, could be id, in my case I chose the run)

Django signal based on the datetime field value

I'm struggling with the following.
I'm trying to create a custom signal that will trigger when the current time will be equal to the value of my model's notify_on DateTimeField.
Something like this:
class Notification(models.Model):
...
notify_on = models.DateTimeField()
def send_email(*args, **kwargs):
# send email
signals.when_its_time.connect(send_email, sender=User)
After I've read through all docs and I found no information on how to implement such a signal.
Any ideas?
UPDATE:
Less naive approach with ability to discard irrelevant tasks: https://stackoverflow.com/a/55337663/9631956
Ok, thanks to comments by #SergeyPugach I've done the following:
Added a post_save signal that calls a function that adds a task to the celery. apply_async let's you pass eta - estimated time of arrival which can accept DateTimeField directly, that's very convenient.
# models.py
from django.db.models import signals
from django.db import models
from .tasks import send_notification
class Notification(models.Model):
...
notify_on = models.DateTimeField()
def notification_post_save(instance, *args, **kwargs):
send_notification.apply_async((instance,), eta=instance.notify_on)
signals.post_save.connect(notification_post_save, sender=Notification)
And the actual task in the tasks.py
import logging
from user_api.celery import app
from django.core.mail import send_mail
from django.template.loader import render_to_string
#app.task
def send_notification(self, instance):
try:
mail_subject = 'Your notification.'
message = render_to_string('notify.html', {
'title': instance.title,
'content': instance.content
})
send_mail(mail_subject, message, recipient_list=[instance.user.email], from_email=None)
except instance.DoesNotExist:
logging.warning("Notification does not exist anymore")
I will not get into details of setting up celery, there's plenty of information out there.
Now I will try to figure out how to update the task after it's notification instance was updated, but that's a completely different story.
In django's documentation there is two interesting signals that may help you on this task: pre_save and post_save.
It depends on your needs, but let's say you want to check if your model's notify_on is equal to the current date after saving your model (actually after calling the save() or create() method). If it's your case you can do:
from datetime import datetime
from django.contrib.auth.models import User
from django.db import models
from django.dispatch import receiver
from django.db.models.signals import post_save
class Notification(models.Model):
...
# Every notification is related to a user
# It depends on your model, but i guess you're doing something similar
user = models.ForeignKey(User, related_name='notify', on_delete=models.DO_NOTHING)
notify_on = models.DateTimeField()
...
def send_email(self, *args, **kwargs):
"""A model method to send email notification"""
...
#receiver(post_save, sender=User)
def create_notification(sender, instance, created, **kwargs):
# check if the user instance is created
if created:
obj = Notification.objects.create(user=instance, date=datetime.now().date())
if obj.notify_on == datetime.now().date():
obj.send_email()
And you should know, that django signals won't work by they own only if there is an action that triggers them. What this mean is that Django signals won't loop over your model's instances and perform an operation, but django signals will trigger when your application is performing an action on the model connected to a signal.
Bonus: To perform a loop over your instances and process an action regulary you may need an asyncworker with a Queue database (mostly, Celery with Redis or RabbitMQ).

Django starting id field from 1000

I would like to start my ids on a django model from 1000. I've found this response on Stackoverflow but I am missing something in my implementation because it is not working.
This is my code in apps.py
from django.apps import AppConfig
from django.db.models.signals import post_migrate
from django.db import IntegrityError
from invoice.models import Invoice
def my_callback(sender, **kwargs):
if sender.name =="invoice":
try:
Invoice.objects.create(id=999)
Invoice.objects.delete()
except IntegrityError:
pass
class InvoiceConfig(AppConfig):
name = 'invoice'
def ready(self):
post_migrate.connect(my_callback, sender=self)
I've then ensure migrate takes place but the model continues to increment from low numbers. What am I missing?

Update Django Model Field Based On Celery Task Status

In my model, I have a status field with a default value of 'Processing'. In the Django admin interface, after user clicks 'Save' button, the form inputs are passed to a celery task that just sleeps for 30 seconds.
After that 30 seconds, how do I:
determine if the celery task was successful?
update the model's status field from 'Processing' to the actual status (ex: Completed, Failed?
models.py
from django.db import models
class Scorecard(models.Model):
name = models.CharField(max_length=100, unique=True)
status = models.CharField(max_length=20, default='Processing')
def __str__(self):
return self.name
admin.py
from django.contrib import admin
from scorecards.models import Scorecard
from scorecards.tasks import get_report
class ScorecardAdmin(admin.ModelAdmin):
list_display = ['name', 'status']
def save_model(self, request, obj, form, change):
if form.is_valid():
data = form.cleaned_data
name = data['name']
get_report.delay(name)
super().save_model(request, obj, form, change)
admin.site.register(Scorecard, ScorecardAdmin)
tasks.py
from __future__ import absolute_import, unicode_literals
from celery import shared_task
from time import sleep
#shared_task
def get_report(name):
sleep(30)
Real-time status of the celery task updating the status field every x time intervals would be nice, but for now I'm just really curious how to do this at all.
I didn't figure out real-time status (yet) but did manage to change the status once a task is completed.
These were the main parts below. The critical reason to understand why this works is that I'm starting a celery worker in --pool=solo like so:
celery -A scorecard worker --pool=solo -l info
This is a single-threaded execution pool (which is fine for my current purposes) but it means it will process the first task get_report(name), and when that's done, process the set_task_status(id) where it checks the status of the result and sets the status field to be whatever the actual status is.
models.py
class Scorecard(models.Model):
....
task_id = models.CharField(max_length=50)
admin.py
class ScorecardAdmin(admin.ModelAdmin):
...
result = get_report.delay(name)
set_task_status.delay(result.task_id)
...
tasks.py
#shared_task
def get_report(name):
sleep(30)
#shared_task
def set_task_status(id):
instance = Scorecard.objects.get(task_id=id)
task_status = AsyncResult(id)
instance.status = task_status.status
instance.save()
This is what I've figured out so far.