In Django project, have a supervisord than start apscheduler
[program:apscheduler]
command=/home/user/Project/.venv/bin/python manage.py runapscheduler
In apscheduler I have one job:
# runapscheduler.py
import logging
import sys
from django.conf import settings
from django.core import management
from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.triggers.cron import CronTrigger
from django.core.management.base import BaseCommand
from django_apscheduler.jobstores import DjangoJobStore
from django_apscheduler.models import DjangoJobExecution
from django_apscheduler import util
logger = logging.getLogger(__name__)
def my_management_command():
management.call_command('MyCommand')
# The `close_old_connections` decorator ensures that database connections, that have become unusable or are obsolete,
# are closed before and after our job has run.
#util.close_old_connections
def delete_old_job_executions(max_age=604_800):
"""
This job deletes APScheduler job execution entries older than `max_age` from the database. It helps to prevent the
database from filling up with old historical records that are no longer useful.
:param max_age: The maximum length of time to retain historical job execution records. Defaults
to 7 days.
"""
DjangoJobExecution.objects.delete_old_job_executions(max_age)
class Command(BaseCommand):
help = "Runs APScheduler."
def handle(self, *args, **options):
scheduler = BlockingScheduler(timezone=settings.TIME_ZONE)
scheduler.add_jobstore(DjangoJobStore(), "default")
scheduler.add_job(
my_management_command,
trigger=CronTrigger(hour="*", minute="*"), # Every hour
id="MyCommand", # The `id` assigned to each job MUST be unique
max_instances=1,
replace_existing=True,
)
logger.info("Added hourly job 'my_management_command'.")
scheduler.add_job(
delete_old_job_executions,
trigger=CronTrigger(
day_of_week="mon", hour="00", minute="00"
), # Midnight on Monday, before start of the next work week.
id="delete_old_job_executions",
max_instances=1,
replace_existing=True,
)
logger.info(
"Added weekly job: 'delete_old_job_executions'."
)
try:
logger.info("Starting scheduler...")
scheduler.start()
except KeyboardInterrupt:
logger.info("Stopping scheduler...")
scheduler.shutdown()
logger.info("Scheduler shut down successfully!")
In my management command "MyCommand", I open some tcp socket to other server.
If I run it outside of apscheduler, the socket are closed correctly once the management command is over.
When it is run with apscheduler socket never close until I restart the job.
Any idea how to fix that ?
Related
I have setup a scheduled task to run daily on PythonAnywhere.
The task uses the Django Commands as I found this was the preferred method to use with PythonAnywhere.
The tasks produces no errors but I don't get any output. 2022-06-16 22:56:13 -- Completed task, took 9.13 seconds, return code was 0.
I have tried uses Print() to debug areas of the code but I cannot produce any output in either the error or server logs. Even after trying print(date_today, file=sys.stderr).
I have set the path on the Scheduled Task as: (Not sure if this is correct but seems to be the only way I can get it to run without errors.)
workon advancementvenv && python3.8 /home/vicinstofsport/advancement_series/manage.py shell < /home/vicinstofsport/advancement_series/advancement/management/commands/schedule_task.py
I have tried setting the path as below but then it gets an error when I try to import from the models.py file (I know this is related to a relative import but cannot management to resolve it). Traceback (most recent call last): File "/home/vicinstofsport/advancement_series/advancement/management/commands/schedule_task.py", line 3, in <module> from advancement.models import Bookings ModuleNotFoundError: No module named 'advancement'
2022-06-17 03:41:22 -- Completed task, took 14.76 seconds, return code was 1.
Any ideas on how I can get this working? It all works fine locally if I use the command py manage.py scheduled_task just fails on PythonAnywhere.
Below is the task code and structure of the app.
from django.core.management.base import BaseCommand
import requests
from advancement.models import Bookings
from datetime import datetime, timedelta, date
import datetime
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail
from django.core.mail import send_mail
import os
from decouple import config
class Command(BaseCommand):
help = 'Sends Program Survey'
def handle(self, *args, **kwargs):
# Get today's date
date_today = datetime.datetime.now().date()
# Get booking data
bookings = Bookings.objects.all()
# For each booking today, send survey email
for booking in bookings:
if booking.booking_date == date_today:
if booking.program_type == "Sport Science":
booking_template_id = 'd-bbc79704a31a4a62a5bfea90f6342b7a'
email = booking.email
booking_message = Mail(from_email=config('FROM_EMAIL'),
to_emails=[email],
)
booking_message.template_id = booking_template_id
try:
sg = SendGridAPIClient(config('SG_API'))
response = sg.send(booking_message)
except Exception as e:
print(e)
else:
booking_template_id = 'd-3167927b3e2146519ff6d9035ab59256'
email = booking.email
booking_message = Mail(from_email=config('FROM_EMAIL'),
to_emails=[email],
)
booking_message.template_id = booking_template_id
try:
sg = SendGridAPIClient(config('SG_API'))
response = sg.send(booking_message)
except Exception as e:
print(e)
else:
print('No')
Thanks in advance for any help.
Thanks Filip and Glenn, testing within the bash console and changing the directory in the task helped to fix the issue. Adding 'cd /home/vicinstofsport/advancement_series && to my task allowed the function to run.'
These are my files-
from django.apps import AppConfig
class ApiConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'api'
def ready(self):
import api.scheduler as scheduler
scheduler.start()
from apscheduler.schedulers.background import BackgroundScheduler
def fetch_new_raw_data():
'''Fetches new data'''
def start():
scheduler = BackgroundScheduler()
scheduler.add_job(fetch_new_raw_data, 'interval', minutes=1)
scheduler.start()
fetch_new_raw_data()
When using py manage.py runserver django spawns 2 processes, each one will start a scheduler.
Is there a way to load up the scheduler only in 1 process and use the same in both or is it ok for them to start their own scheduler?
In production mode or debug mode, this class is initialized multiple times. this solution is not working very well, but you can record os.getpid() result in a file or DB record and check it every time. If PID is changed you start scheduler again.
import os
from django.apps import AppConfig
class ApiConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'api'
def ready(self):
pid = os.getpid()
with open("/path/to/log/api_pid", "r") as pid_file:
if pid_file.read().strip() == pid:
return
with open("/path/to/log/api_pid", "w") as pid_file:
pid_file.write(pid)
import api.scheduler as scheduler
scheduler.start()
But I recommend you to use external services like celery project or use management commands and run it after runing the project.
I'm using Celery 4.4 with Django 2.2
I have to create a Periodic Task, I'm extending PeriodicTask ask as
from celery.schedules import crontab
from celery.task import PeriodicTask
class IncompleteOrderHandler(PeriodicTask):
run_every = crontab(
minute='*/{}'.format(getattr(settings, 'INCOMPLETE_ORDER_HANDLER_PULSE', 5))
)
def run(self, *args, **kwargs):
# Task definition
eligible_users, slot_begin, slot_end = self.get_users_in_last_slot()
map(lambda user: self.process_user(user, slot_begin, slot_end), eligible_users)
Earlier to register the above task, I used to call
from celery.registry import tasks
tasks.register(IncompleteOrderHandler)
But now there is no registry module in the celery. How can I register the above periodic task?
I had the same problem with class based celery tasks. This has to works, but it doesn't!
Accidentally, my problem solved by every one on these two changes:
I import one of the class based tasks at tasks.py in viewsets.py, and suddenly i figured out that after doing that, celery found all of the tasks at tasks.py.
This was my base celery setting file:
from __future__ import absolute_import
import os
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'picha.settings')
app = Celery('picha')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
I changed the last line to app.autodiscover_tasks(lambda:
settings.CELERY_TASKS) and add CELERY_TASKS list to settings.py and write all
tasks.py file paths in it and then celery found tasks.
I hope one of these work for you.
using celery with SQS in flask app
but celery is receiving same task twice with same task id at same time,
running worker like this,
celery worker -A app.jobs.run -l info --pidfile=/var/run/celery/celery.pid --logfile=/var/log/celery/celery.log --time-limit=7200 --concurrency=8
here are the logs of celery
[2019-11-29 08:07:35,464: INFO/MainProcess] Received task: app.jobs.booking.bookFlightTask[657985d5-c3a3-438d-a524-dbb129529443]
[2019-11-29 08:07:35,465: INFO/MainProcess] Received task: app.jobs.booking.bookFlightTask[657985d5-c3a3-438d-a524-dbb129529443]
[2019-11-29 08:07:35,471: WARNING/ForkPoolWorker-4] in booking funtion1
[2019-11-29 08:07:35,473: WARNING/ForkPoolWorker-3] in booking funtion1
[2019-11-29 08:07:35,537: WARNING/ForkPoolWorker-3] book_request_pp
[2019-11-29 08:07:35,543: WARNING/ForkPoolWorker-4] book_request_pp
same task received twice and both are running simultaneously,
using celery==4.4.0rc4 , boto3==1.9.232, kombu==4.6.6 with SQS in python flask.
In SQS, Default Visibility Timeout is 30 minutes, and my task is not having ETA and not ack
my task.py
from app import app as flask_app
from app.jobs.run import capp
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy(flask_app)
class BookingTasks:
def addBookingToTask(self):
request_data = request.json
print ('in addBookingToTask',request_data['request_id'])
print (request_data)
bookFlightTask.delay(request_data)
return 'addBookingToTask added'
#capp.task(max_retries=0)
def bookFlightTask(request_data):
task_id = capp.current_task.request.id
try:
print ('in booking funtion1')
----
my config file, config.py
import os
from urllib.parse import quote_plus
aws_access_key = quote_plus(os.getenv('AWS_ACCESS_KEY'))
aws_secret_key = quote_plus(os.getenv('AWS_SECRET_KEY'))
broker_url = "sqs://{aws_access_key}:{aws_secret_key}#".format(
aws_access_key=aws_access_key, aws_secret_key=aws_secret_key,
)
imports = ('app.jobs.run',)
## Using the database to store task state and results.
result_backend = 'db' + '+' + os.getenv('SQLALCHEMY_DATABASE_URI')
and lastly my celery app file, run.py
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from flask import Flask
from app import app as flask_app
import sqlalchemy
capp = Celery()
capp.config_from_object('app.jobs.config')
# Optional configuration, see the capplication user guide.
capp.conf.update(
result_expires=3600,
)
# SQS_QUEUE_NAME is like 'celery_test.fifo' , .fifo is required
capp.conf.task_default_queue = os.getenv('FLIGHT_BOOKINNG_SQS_QUEUE_NAME')
if __name__ == '__main__':
capp.start()
The default SQS visiblity_timeout is 30s. You need to update the celery config value:
broker_transport_options={'visibility_timeout': 3600}.
When celery goes to create the queue it will set the visibility timeout to 1h.
NOTE: If you specify the task_default_queue, and the queue has already been created without specifying broker_transport_options={'visibility_timeout': 3600}, celery will not update the visibility timeout when restarted with broker_transport_options={'visibility_timeout': 3600}. You will need to delete the queue and have celery recreate it.
How to write specific log file for specific task in django celery,i have write overall celery log in single log file
how to do that one
from redington.celery import app
from celery import shared_task
#app.task(bind=True, name='load_ibm_machine_images')
def load_ibm_machine_images(self):
""" Task to perform To load soft-layer machine images """
from cloudapp.management.commands.update_softlayer_machine_types import Command
soft_layer_command_object = Command()
soft_layer_command_object.handle()
return True
Thanks in advance!!!