Pesky server error on Python web app - django

I've been having a problem with a website that I'm working on not attaching pictures to html emails. Thought I had it fixed but then every time someone tries to register on it I get a Server Error (500). I've only changed a couple of references so no idea what went wrong there, anyways error log is as follows:
2016-08-31 08:26:15,757 :Internal Server Error: /register/
Traceback (most recent call last):
File "/home/asranet/.virtualenvs/testenv/local/lib/python2.7/site-packages/django/core/handlers/base.py", line 149, in get_response
response = self.process_exception_by_middleware(e, request)
File "/home/asranet/.virtualenvs/testenv/local/lib/python2.7/site-packages/django/core/handlers/base.py", line 147, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "./register/views.py", line 14, in index
form.save(commit=True)
File "/home/asranet/.virtualenvs/testenv/local/lib/python2.7/site-packages/django/forms/models.py", line 451, in save
self.instance.save()
File "./register/models.py", line 35, in save
email_client(self, site_settings.site_name + "Conference Registration", "You are officially registered for AdWind 2017")
File "./adWind/email_functionality.py", line 31, in email_client
fp = open(os.path.join(os.path.dirname(__file__), f), 'rb')
IOError: [Errno 2] No such file or directory: u'./adWind/static/Images/asranetLogo.jpg'
I checked and the file is there. No idea how to proceed, could really use some help. Thank you in advance!
P.S. here's the code for email functionality:
from __future__ import unicode_literals
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string
import os
from email.mime.image import MIMEImage
from reportlab.platypus.doctemplate import SimpleDocTemplate
from reportlab.platypus import Paragraph, Spacer
from reportlab.lib.styles import getSampleStyleSheet
from django.http import HttpResponse
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
def email_client(self, subject, text):
# Send the client an email
html_content = render_to_string("../templates/baseTemplates/emailToUser.html", {'salutation': self.salutation,
'last_name':
self.last_name,
'text_body': text})
msg = EmailMultiAlternatives(subject, 'Dear ' + self.salutation + ' ' +
self.last_name + '/n' + text,
'adwind#asranet.co.uk', [self.email], )
msg.attach_alternative(html_content, "text/html")
msg.attach_file('/static/Images/asranetLogo.jpg')
msg.mixed_subtype = 'related'
f = '/static/Images/asranetLogo.jpg'
fp = open(os.path.join(os.path.dirname(__file__), f), 'rb')
msg_img = MIMEImage(fp.read())
fp.close()
msg_img.add_header('Content-ID', '<{}>'.format(f))
msg.attach(msg_img)
msg.send(fail_silently=True)
def email_admin(self, subject, text, sorted_self):
styleSheet = getSampleStyleSheet()
# Send the admin a PDF of client details
response = HttpResponse(content_type='application/pdf')
response['Content-Disposition'] = 'attachment; filename="clientDetails.pdf"'
string_buffer = StringIO()
new_pdf = []
header = Paragraph("AdWind Attendee Details", styleSheet['Heading1'])
new_pdf.append(header)
for element in sorted_self:
new_pdf.append(Paragraph(element[0], styleSheet['Heading3']))
new_pdf.append(Paragraph(element[1], styleSheet['BodyText']))
new_pdf.append(Spacer(1, 2))
doc = SimpleDocTemplate(string_buffer)
doc.build(new_pdf)
pdf = string_buffer.getvalue()
string_buffer.close()
msg = EmailMultiAlternatives(subject, text, "adwind#asranet.co.uk", ["adwind#asranet.co.uk"])
msg.attach(self.first_name + self.last_name + "adWind.pdf", pdf, "application/pdf")
msg.send(fail_silently=True)

In Django the static directory is usually under a module_name directory. I suggest changing your paths into a relative one:
msg.attach_file('static/[module_name]/Images/asranetLogo.jpg')
and
f = 'static/[module_name]/Images/asranetLogo.jpg'

Related

runtime error. there is no current event loop in thread 'Thread-1'

i followed this tutorial https://python.gotrained.com/scraping-telegram-group-members-python-telethon/, and the code ran successfully on a single python script. so i decided to implement the same code into django views.py. form.py, html templates and urls.py are properly connected. whenever i runserver and enter details it raises runtime error saying "There is no current event loop in thread 'Thread-1'."
views.py
import asyncio
from django.shortcuts import render,redirect
from telethon.sync import TelegramClient
from .form import DetailForm,CodeForm
from .models import Detail,Code
# Create your views here.
def home(request):
form = DetailForm
context = {'form':form}
if request.method=='POST':
form = DetailForm(request.POST)
if form.is_valid():
form.save()
details = Detail.objects.all().last()
api_id = int(details.api_id)
api_hash = details.api_hash
phone_number = details.phone_number
loop = asyncio.new_event_loop()
client = TelegramClient(phone_number, api_id, api_hash, loop=loop)
client.connect()
if not client.is_user_authorized():
client.send_code_request(phone_number)
# global client, phone_number
return redirect('code')
return render(request, 'teleapp/home.html', context)
def codeview(request):
code=CodeForm
# codemodel = Code.objects.all().last()
if request.method == 'POST':
codes = CodeForm(request.POST)
if codes.is_valid():
codes.save()
#what next...
return redirect('scrapy')
context = {'code':code}
return render(request, 'teleapp/code.html', context)
def scrappy(request):
details = Detail.objects.all().last()
context = {'details':details}
return render(request, 'teleapp/scrappy.html', context)```
**error raised**
Internal Server Error: /
Traceback (most recent call last):
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\User\PycharmProjects\telegram\scrapy\teleapp\views.py", line
24, in home
client.connect()
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\telethon
\sync.py", line 35, in syncified
loop = asyncio.get_event_loop()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\asyncio\ev
ents.py", line 694, in get_event_loop
return get_event_loop_policy().get_event_loop()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\asyncio\events.py", line 602, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-2'.

PIL module ImportError on Apache2 Server

I am getting this traceback on my Apache2 server when trying to upload an image. I am using Python 3.5.2 and Pillow 5.0.0 and Django 2.1.1
I was able to upload images successfully when I am working on my localhost, but when I uploaded this code to the apache2 server, then it gives this traceback. Pillow is in the requiremnts.txt though. No where in the code do I explicitly try to import PIL, but it seems to be getting called.
I have tried installing Pillow and PIL. I have tried to uninstall PIL, uninstall Pillow, I have tried re-install both. Nothing seems to be working.
View.py
from django.http import JsonResponse, HttpResponse
from django.shortcuts import render
from django.core import serializers
from manager.models import Manager, Inspection
from . import InspectorService
from . import InspectionService
import json
from .forms import InspectionImageForm
... more unrelevant code here ...
def upload_image(request):
if request.method == 'POST':
form = InspectionImageForm(request.POST, request.FILES)
inspection_id = request.POST.get('inspection_id')
if form.is_valid():
m = Inspection.objects.get(id = inspection_id)
m.image = form.cleaned_data['image']
m.save()
return JsonResponse({'error': False, 'message': 'Uploaded Successfully', 'location': m.image.name})
else:
return JsonResponse({'error': True, 'errors': form.errors})
else:
return JsonResponse({'error': True, 'errors': 'Post error'})
forms.py
from django import forms
class InspectionImageForm(forms.Form):
image = forms.ImageField()
inspection_id = forms.IntegerField()
Traceback
File "/opt/myenv/lib/python3.5/site-packages/django/core/handlers/exception.py" in inner
34. response = get_response(request)
File "/opt/myenv/lib/python3.5/site-packages/django/core/handlers/base.py" in _get_response
126. response = self.process_exception_by_middleware(e, request)
File "/opt/myenv/lib/python3.5/site-packages/django/core/handlers/base.py" in _get_response
124. response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/var/www/html/CMPUT401/computronix/manager/views.py" in upload_image
119. if form.is_valid():
File "/opt/myenv/lib/python3.5/site-packages/django/forms/forms.py" in is_valid
185. return self.is_bound and not self.errors
File "/opt/myenv/lib/python3.5/site-packages/django/forms/forms.py" in errors
180. self.full_clean()
File "/opt/myenv/lib/python3.5/site-packages/django/forms/forms.py" in full_clean
381. self._clean_fields()
File "/opt/myenv/lib/python3.5/site-packages/django/forms/forms.py" in _clean_fields
397. value = field.clean(value, initial)
File "/opt/myenv/lib/python3.5/site-packages/django/forms/fields.py" in clean
584. return super().clean(data)
File "/opt/myenv/lib/python3.5/site-packages/django/forms/fields.py" in clean
147. value = self.to_python(value)
File "/opt/myenv/lib/python3.5/site-packages/django/forms/fields.py" in to_python
613. from PIL import Image
Exception Type: ImportError at /manager/upload_image/
Exception Value: No module named 'PIL'

scrapy TypeError: object() takes no parameters

I am new to Scrapy and trying to crawl a couple of links as a test using Scrapy. Whenever I run scrapy crawl tier1, I get "TypeError: object() takes no parameters" as the following:
Traceback (most recent call last):
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/adv_crawler/adv_crawler/spiders/tier1_crawler.py", line 93, in parse
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 167, in add_xpath
self.add_value(field_name, values, *processors, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 77, in add_value
self._add_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 91, in _add_value
processed_value = self._process_input_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 150, in _process_input_value
return proc(value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/processors.py", line 28, in __call__
next_values += arg_to_iter(func(v))
TypeError: object() takes no parameters
2017-08-23 17:25:02 [tier1-parse-logger] INFO: Entered the parse function to parse and index: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [tier1-parse-logger] ERROR: Error (object() takes no parameters) when trying to parse <<date>> from a mk article: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [tier1-parse-logger] ERROR: Error (object() takes no parameters) when trying to parse <<author>> from a mk article: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [scrapy.core.scraper] ERROR: Spider error processing <GET http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166> (referer: None)
Traceback (most recent call last):
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/adv_crawler/adv_crawler/spiders/tier1_crawler.py", line 93, in parse
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 167, in add_xpath
self.add_value(field_name, values, *processors, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 77, in add_value
self._add_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 91, in _add_value
processed_value = self._process_input_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 150, in _process_input_value
return proc(value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/processors.py", line 28, in __call__
next_values += arg_to_iter(func(v))
TypeError: object() takes no parameters
And, my spider file (tier1_crawler.py):
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import os
sys.path.append(os.path.abspath('..'))
import logging
import scrapy
from scrapy.loader import ItemLoader
from adv_crawler.items import AdvCrawlerItem
from datetime import datetime, date, time
t1_parse_logger = logging.getLogger("tier1-parse-logger")
t1_parse_logger.LOG_FILE = "Tier1-log.txt"
content_type_dic = {
'news': 'news',
}
class Tier1Crawler(scrapy.Spider):
name = "tier1"
def start_requests(self):
urls = ['http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535982',
'http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166',
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
t1_parse_logger.info("Entered the parse function to parse and index: %s" % response.url) # Log at the beginning of the parse function
item_loader = ItemLoader(item=AdvCrawlerItem(), response=response)
if 'mk.co.kr' in response.url:
mk_loader = item_loader.nested_xpath('//div[#id="top_header"]/div[#class="news_title"]/div[#class="news_title_text"]')
try:
mk_loader.add_xpath('date', 'div[#class="news_title_author"]/ul/li[#class="lasttime"]')
except AttributeError: # if the date is not in "lasttime" li tag
mk_loader.add_xpath('date', 'div[#class="news_title_author"]/ul/li[#class="lasttime1"]')
except Exception as e: # in case the error is not AttributeError
t1_parse_logger.error("Error "+"("+str(e)+")"+" when trying to parse <<date>> from a mk article: %s" % response.url)
try:
mk_loader.add_xpath('author', 'div[#class="news_title_author"]/ul/li[#class="author"]')
except AttributeError: # in case there is no author (some mk articles have no author)
item_loader.add_value('author', "None") # ir error, replace with the line below
# item['author'] = "None" # if the above gives any error, replace the above with this line
except Exception as e: # in case the error is not AttributeError
t1_parse_logger.error("Error "+"("+str(e)+")"+" when trying to parse <<author>> from a mk article: %s" % response.url)
item_loader.add_xpath('content', '//div[#id="Content"]/div[#class="left_content"]/div[#id="article_body"]/div[#class="art_txt"]') # Content of the article (entire contents)
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
item_loader.add_value('content_type', content_type_dic['news'])
item_loader.add_value('timestamp', str(datetime.now())) # timestamp of when the document is being indexed
item_loader.add_value('url', response.url) # url of the article
t1_parse_logger.info("Parsed and indexed: %s" % response.url)
return item_loader.load_item()
And, my items.py file:
# -*- coding: utf-8 -*-
import scrapy
from scrapy.loader.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_date(value):
if isinstance(value, unicode):
(year, month, day) = str(value.split(" ")[-2]).split(".")
return year+"-"+month+"-"+day
def filter_utf(value):
if isinstance(value, unicode):
return value.encode('utf-8')
class AdvCrawlerItem(scrapy.Item):
author = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_utf),) # Name of the publisher/author
content = scrapy.Field(input_processor=MapCompose(remove_tags, Join, filter_utf),) # Content of the article (entire contents)
content_type = scrapy.Field()
date = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_date),)
timestamp = scrapy.Field() # timestamp of when the document is being indexed
title = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_utf),) # title of the article
url = scrapy.Field() # url of the article
And, pipelines.py file:
import json
from scrapy import signals
from scrapy.exporters import JsonLinesItemExporter
class AdvCrawlerJsonExportPipeline(object):
def open_spider(self, spider):
self.file = open('crawled-articles1.txt', 'w')
def close_spider(self, spider):
self.file.close()
def process_item(self, item, spider):
line = json.dummps(dict(item)) + "\n"
self.file.write(line)
return item
I am aware that "TypeError: object() takes no parameters" error is usually thrown when __init__ method of a class is not defined at all or not defined to take in parameter(s).
However, in the case above, how can i fix the error? Am I doing something wrong using the item loader or nested item loader??
When using scrapy processors you should use the classes to create objects that do the processing:
# wrong
field = Field(output_processor=MapCompose(TakeFirst))
# right
field = Field(output_processor=MapCompose(TakeFirst()))
^^

ValueError: Missing scheme in request url: h

I am a beginner in scrapy, python. I tried to deploy the spider code in scrapinghub and I encountered the following error. Below is the code.
import scrapy
from bs4 import BeautifulSoup,SoupStrainer
import urllib2
from scrapy.selector import Selector
from scrapy.http import HtmlResponse
import re
import pkgutil
from pkg_resources import resource_string
from tues1402.items import Tues1402Item
data = pkgutil.get_data("tues1402","resources/urllist.txt")
class SpiderTuesday (scrapy.Spider):
name = 'tuesday'
self.start_urls = [url.strip() for url in data]
def parse(self, response):
story = Tues1402Item()
story['url'] = response.url
story['title'] = response.xpath("//title/text()").extract()
return story
is my spider.py code
import scrapy
class Tues1402Item(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
url = scrapy.Field()
is the items.py code and
from setuptools import setup, find_packages
setup(
name = 'tues1402',
version = '1.0',
packages = find_packages(),
entry_points = {'scrapy': ['settings = tues1402.settings']},
package_data = {'tues1402':['resources/urllist.txt']},
zip_safe = False,
)
is the setup.py code.
The following is the error.
Traceback (most recent call last):
File "/usr/local/lib/python2.7/site-packages/scrapy/core/engine.py", line 126, in _next_request
request = next(slot.start_requests)
File "/usr/local/lib/python2.7/site-packages/scrapy/spiders/init.py", line 70, in start_requests
yield self.make_requests_from_url(url)
File "/usr/local/lib/python2.7/site-packages/scrapy/spiders/init.py", line 73, in make_requests_from_url
return Request(url, dont_filter=True)
File "/usr/local/lib/python2.7/site-packages/scrapy/http/request/init.py", line 25, in init
self._set_url(url)
File "/usr/local/lib/python2.7/site-packages/scrapy/http/request/init.py", line 57, in _set_url
raise ValueError('Missing scheme in request url: %s' % self._url)
ValueError: Missing scheme in request url: h
Thank you in Advance
Your error means that url h is not a valid url. You should print out your self.start_urls and see what urls you have there, you most likely have a string h as your first url.
Seems like your spider iterates through text instead of a list of urls here:
data = pkgutil.get_data("tues1402","resources/urllist.txt")
class SpiderTuesday (scrapy.Spider):
name = 'tuesday'
self.start_urls = [url.strip() for url in data]
Assuming that you store your urls with some separator in urllist.txt file you should split that:
# assuming file has url in every line
self.start_urls = [url.strip() for url in data.splitlines()]

testing django web app that uses cookies/session

In views.py:
get_dict = Site.objects.getDictionary(request.COOKIES['siteid'])
{gets a dictionary with site information based on id from cookie}
In tests.py:
from django.test import TestCase
class WebAppTest(TestCase):
def test_status(self):
response = self.client.get('/main/',{})
response.status_code # --->passed with code 200
response = self.client.get('/webpage/',{'blog':1})
response.status_code # ----> this is failing
In order to present blog page it goes to a view where it gets a dictionary using existing cookie, process it, renders templates, which works fine when running the app. But the tests are failing.Having never tested Django webapps I'm not sure how to test it right. Here is the traceback.
Traceback (most recent call last):
File "<console>", line 2, in <module>
File "/usr/lib/pymodules/python2.6/django/test/client.py", line 313, in post
response = self.request(**r)
File "/usr/lib/pymodules/python2.6/django/core/handlers/base.py", line 92, in get_response
response = callback(request, *callback_args, **callback_kwargs)
File "/var/lib/django/data/../webpage/views.py", line 237, in getCostInfo
get_dict = Site.objects.getDictionary(request.COOKIES['siteid'])
KeyError: 'siteid'
Went through some online samples but couldn't find something that deals in depth with cookies/sessions. Any ideas or directs to useful links are highly appreciated.
Take a look at the Persistent State section of the Django Testing docs.
In your case, I would expect your test to be something more like:
from django.test import TestCase
from django.test.client import Client
class WebAppTest(TestCase):
def setUp(self):
self.client = Client()
session = self.client.session
session['siteid'] = 69 ## Or any valid siteid.
session.save()
def test_status(self):
response = self.client.get('/main/',{})
self.assertEqual(response.status_code, 200)
response = self.client.get('/webpage/',{'blog':1})
self.assertEqual(response.status_code, 200)