How to run scrapy spider with celery - django

Question

I'm trying to run a scrapy spider which takes some argument and runing it with os.system. But the celery task(scraper) doesn't gets executed untill it finishes.
Spider
class SpecificAuthorQuotesSpider(scrapy.Spider):
"""Extracts the quotes from specific author"""
start_urls = ['https://quotes.toscrape.com/']
name = "some-quotes"
def __init__(self, author=None, **kwargs):
self.author = author
super().__init__(**kwargs)
def parse(self, response, **kwargs):
item = QuotesItem()
all_div_quotes = response.css('div.quote')
for quote in all_div_quotes:
title = quote.css('span.text::text').extract_first().replace('”', '').replace("“", "")
author = quote.css('.author::text').extract_first()
# Check if author's name matches
if author.strip().lower() == self.author.strip().lower():
item['text'] = title
item['author'] = author
yield item
# Crawl Next Page
next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
Task
#shared_task
def task_scrape_from_author(author_name):
"""Scrape quotes from author"""
django_path = Path(__file__).resolve().parent.parent
os.chdir(str(django_path)+"/scraper")
os.system(
"scrapy crawl some-quotes -a author='{}'".format(author_name))
View
def scrape_quotes_from_author(request):
if request.user.is_superuser:
author_name = request.POST.get("athr_name")
task_scrape_from_author.delay(author_name)
messages.add_message(
request, messages.INFO, 'Started crawling quotes from {}'.format(author_name))
return HttpResponseRedirect(reverse("admin:index"))
else:
return HttpResponseRedirect("../")
Github Repo
I don't understand why is the task not getting completed and interrupted without any messages. I tried setting max timeout also but that din't worked.

Answer 1

I make a table for spider_name and spider_class.
model.py
class Spiders(models.Model):
spider_class = models.CharField(max_length=50,verbose_name="Spider Class",null=True)
spider_name = models.CharField(max_length=50,verbose_name="Spider Name",null=True)
I collect all spider_name and class' here.
view.py
from .model import Spiders
from spider_dir.start import startallSpiders
def runAllspiders(request):
all_class = []
spiders = Spiders.objects.all()
for spider in spiders:
spider_name = spider.spider_name
name = 'spider_dir.spider_dir.spiders.'+spider_name
i = importlib.import_module(name)
class_ = getattr(i, spider.spider_class)
all_class.append(class_)
try:
startallSpiders(all_class)
messages.success(request,"Spiders works fine")
except:
messages.warning(request,"An error occure")
return redirect(request.META['HTTP_REFERER'])
I make a start py in scrapy dir
I use crochet to start all of the spiders at once.
start.py
from .spider_dir import settings as st
from scrapy.settings import Settings
from crochet import setup
setup()
def startallSpiders(all_Class):
for class_ in all_Class:
crawler_settings = Settings()
setup()
crawler_settings.setmodule(st)
runner= CrawlerRunner(settings=crawler_settings)
runner.crawl(class_)
settings.py You have to append Django settings in scrapy settings.
import os,sys
sys.path.append(os.path.dirname(os.path.abspath('.')))
os.environ['DJANGO_SETTINGS_MODULE'] = 'django_project.settings'
import django
django.setup()
I figure with Crawlerrunner and everything works fine for 6 months.