Setting variable is not being set up during testing middleware (p - django

settings.py
MY_VAR = os.get("MY_VAR", False)
custom_middleware.py
from my_proj.settings import MY_VAR
from django.core.exceptions import MiddlewareNotUsed
class CustomMiddleware:
def _init_(self, get_response):
if MY_VAR == 'False':
raise MiddlewareNotUsed
self.get_response = get_response
def __call__(self, request):
if MY_VAR == 'True':
#My custom logic
return
response = self.get_response(request)
return response
test_custom_middleware.py
import os from unittest.mock import Mock
from api.middlewares.custom_middleware import CustomMiddleware
class TestLCustomMiddleware:
def test(self, settings):
request = Mock()
settings.MY_VAR = 'True'
assert settings.MY_VAR
with patch.dict('os.environ', {'MY_VAR': 'True'}):
assert 'MY_VAR' in os.environ
middleware = CustomMiddleware(get_response='response')
middleware(request)
In CustomMiddleware I always get "False" in MY_VAR variable of settings. How can I set it up?
Updated.
Thanks to Philippe and Dharman.
It works fine
custom_middleware.py
from django.conf import settings
from django.core.exceptions import MiddlewareNotUsed
class CustomMiddleware:
def _init_(self, get_response):
if settings.MY_VAR == 'False':
raise MiddlewareNotUsed
self.get_response = get_response
def __call__(self, request):
if settings.MY_VAR == 'True':
#My custom logic
return
response = self.get_response(request)
return response
test_custom_middleware.py
from api.middlewares.custom_middleware import CustomMiddleware
from django.test import override_settings
class TestLCustomMiddleware:
#override_settings(MY_VAR="True")
def test(self):
request = Mock()
middleware = CustomMiddleware(get_response='response')
middleware(request)

As explaned in Django documentation, you should import your setting using :
from django.conf import settings
if settings.MY_VAR:
# Do something
"django.conf.settings" isn’t a module, it’s an object. So importing individual settings is not possible. This example below won't work properly :
from django.conf.settings import MY_VAR
Furthermore, it's not recommended to change setting during Runtime :
from django.conf import settings
settings.DEBUG = True # Don't do this!
Instead you should try using override_settings() or modify_settings().
For example :
from django.test import TestCase, override_settings
class TestLCustomMiddleware(TestCase):
#override_settings(MY_VAR=True) # without quote
def test(self):
...

Related

How to Test in case Unittest By pass file excel to Import API

i'm will test code by TestCase of django.test and APIClient of resrt_framework.test with excel file, where do i can test thus import api
test_import_file.py
import xlrd
from django.test import TestCase
from rest_framework.test import APIClient
from account.models import Account
from account.tests import create_super_user
from instructor.models import Instructor
from utils.excel import get_value
class TestClassImport(TestCase):
def setUp(self):
self.account = create_super_user()
self.client = APIClient()
self.client.force_authenticate(self.account)
self.url = 'static/example/example_class_import.xlsx'
self.file = open('static/example/example_class_import.xlsx', 'rb')
self.wb = xlrd.open_workbook(file_contents=self.file.read())
self.sh = self.wb.sheet_by_index(0)
def test_real_import(self):
file = open(self.url, encoding="utf8", errors='ignore')
url = '/api/dashboard/content-migration/import/instructor/'
self.response = self.client.post(url, file)
self.failUnlessEqual(self.response.status_code, 201)
I hope it will "test_real_import (class.unittest.test_import_file.TestInstructorImport) ... ok"
add every thing to task of celery and call it by unit test of django
'python'
from your_app import your_project_task
class TestInstructorImport(TestCase):
fixtures = ['instructor/fixtures/instructor.json', 'account/fixtures/account.json']
def setUp(self):
self.account = create_super_user()
self.file_path = '/yourpath/name.xlsx'
def test_task_import(self):
response = task_execute.delay(self.file_path)
self.assertEqual(response.result, 'Done.')
code "self.assertEqual(response.result, 'Done.')"
in task just
return('Done.')
when your code done without any error.

How to turn a variable into a module with sys.modules hack?

import os
from glob import glob
from importlib import import_module
from django.urls import re_path as _re_path, path as _path
def _glob_init(name):
name = name.replace('.', os.sep)
path = os.sep + '**'
modules = []
for module in glob(name + path, recursive=True):
importable = os.path.splitext(module)[0].replace(os.sep, '.')
if '__' in importable:
continue
try:
module = import_module(importable)
except ModuleNotFoundError:
module = import_module(importable[:-1])
modules.append(module)
return modules
class UrlManager:
def __init__(self, views_root):
self.views_root = views_root
self._url_patterns = []
def _path(self, route, kwargs=None, name=None, is_re=None):
func = _re_path if is_re else _path
def decorator(view):
_view = view # keep the original view
if isinstance(view, type):
view = view.as_view()
self._url_patterns.append(
func(route, view, kwargs=kwargs, name=name or view.__name__)
)
return _view
return decorator
def path(self, route, kwargs=None, name=None):
return self._path(route, kwargs=kwargs, name=name, is_re=False)
def re_path(self, route, kwargs=None, name=None):
return self._path(route, kwargs=kwargs, name=name, is_re=True)
#property
def url_patterns(self):
if isinstance(self.views_root, str):
_glob_init(self.views_root)
else:
for root in self.views_root:
_glob_init(root)
return self._url_patterns
Basic usage:
# app.urls.py
app_urls = UrlManager('app.views')
# app.views.py
from models import SomeModel
from urls import app_urls
#app_urls.path('foo/', name='foo')
def view(request):
return response
# project.urls.py
from django.urls import include
from app.urls import app_urls
urlpatterns = [
path('', include(app_urls.urlpatterns))
]
Okay, now:
Everything works when you already have the migrations up and running, but when you want to create a new migration; due the fact that this code flow is something like:
--project urls.py imports app.urls
--app.urls imports views
--views imports models
--models doesn't exist yet because this is a migration command run
--error
I need to somehow either create a module named the instance so I can use include('module.urls') or some other way to postpone the imports.
Repo is at https://github.com/isik-kaplan/django_urls if anyone wants to open a pr.

Run a Scrapy spider in a Celery Task (django project)

I'm trying to run scrapy (spider/crawl)from django project (task in the admin interrface using celery). this is my code .
this is the error when I try to call the task from a python shell
djangoproject:
-monapp:
-tasks.py
-spider.py
-myspider.py '
-models.py
.....
tasks.py:
from djcelery import celery
from demoapp.spider import *
from demoapp.myspider import *
#celery.task
def add(x, y):
return x + y
#celery.task
def scra():
result_queue = Queue()
crawler = CrawlerWorker(MySpider(), result_queue)
crawler.start()
return "success"
spider.py:
from scrapy import project, signals
from scrapy.settings import Settings
from scrapy.crawler import Crawler
from scrapy.xlib.pydispatch import dispatcher
from multiprocessing.queues import Queue
import multiprocessing
class CrawlerWorker(multiprocessing.Process):
def __init__(self, spider, result_queue):
multiprocessing.Process.__init__(self)
self.result_queue = result_queue
self.crawler = Crawler(Settings())
if not hasattr(project, 'crawler'):
self.crawler.install()
self.crawler.configure()
self.items = []
self.spider = spider
dispatcher.connect(self._item_passed, signals.item_passed)
def _item_passed(self, item):
self.items.append(item)
def run(self):
self.crawler.crawl(self.spider)
self.crawler.start()
self.crawler.stop()
self.result_queue.put(self.items)
myspider.py
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.item import Item, Field
class TorentItem(Item):
title = Field()
desc = Field()
class MySpider(CrawlSpider):
name = 'job'
allowed_domains = ['tanitjobs.com']
start_urls = [\
'http://tanitjobs.com/browse-by-category/Nurse/',]
rules = (
Rule (SgmlLinkExtractor(allow=('page=*',)
,restrict_xpaths=('//div[#class="pageNavigation"]',),
unique = True)
, callback='parse_item', follow= True),
)
def parse_item(self, response):
hxs = HtmlXPathSelector(response)
items= hxs.select('\
//div[#class="offre"]/div[#class="detail"]')
scraped_items =[]
for item in items:
scraped_item = TorentItem()
scraped_item['title']=item.select(\
'a/strong/text()').extract()
scraped_item['desc'] =item.select(\
'./div[#class="descriptionjob"]/text()').extract()
scraped_items.append(scraped_item)
return scraped_items
I got it work mine on the shell using django management command. Below is my code snippet. Feel free to modify to fit your needs.
from twisted.internet import reactor
from scrapy.crawler import Crawler
from scrapy import signals
from scrapy.utils.project import get_project_settings
from django.core.management.base import BaseCommand
from myspiderproject.spiders.myspider import MySpider
class ReactorControl:
def __init__(self):
self.crawlers_running = 0
def add_crawler(self):
self.crawlers_running += 1
def remove_crawler(self):
self.crawlers_running -= 1
if self.crawlers_running == 0:
reactor.stop()
def setup_crawler(domain):
settings = get_project_settings()
crawler = Crawler(settings)
crawler.configure()
crawler.signals.connect(reactor_control.remove_crawler, signal=signals.spider_closed)
spider = MySpider(domain=domain)
crawler.crawl(spider)
reactor_control.add_crawler()
crawler.start()
reactor_control = ReactorControl()
class Command(BaseCommand):
help = 'Crawls the site'
def handle(self, *args, **options):
setup_crawler('somedomain.com')
reactor.run() # the script will block here until the spider_closed signal was sent
hope this helps.

cProfiler Django middleware: sorting the stats

In Django, I am using the below middleware Cprofiler snippet /from http://djangosnippets.org/snippets/727/ )
How do I change what is used to sort it? If I want to use sort_stats() where does that go in the code?
import sys
import cProfile
from cStringIO import StringIO
from django.conf import settings
class ProfilerMiddleware(object):
def process_view(self, request, callback, callback_args, callback_kwargs):
if settings.DEBUG and 'prof' in request.GET:
self.profiler = cProfile.Profile()
args = (request,) + callback_args
return self.profiler.runcall(callback, *args, **callback_kwargs)
def process_response(self, request, response):
if settings.DEBUG and 'prof' in request.GET:
self.profiler.create_stats()
out = StringIO()
old_stdout, sys.stdout = sys.stdout, out
self.profiler.print_stats(1)
sys.stdout = old_stdout
response.content = '<pre>%s</pre>' % out.getvalue()
return response
I think you are looking for the sort_stats function and it needs to go right before print_stats.
def process_response(self, request, response):
if settings.DEBUG and 'prof' in request.GET:
self.profiler.create_stats()
out = StringIO()
old_stdout, sys.stdout = sys.stdout, out
self.profiler.sort_stats('name')
self.profiler.print_stats(1)
sys.stdout = old_stdout
response.content = '<pre>%s</pre>' % out.getvalue()
return response
Also take a look at this http://docs.python.org/2/library/profile.html
sort_stats() cannot be used directly on the profiler. I found the solution here:
http://djangosnippets.org/snippets/1579/

How to test 500.html error page in django development env?

I am using Django for a project and is already in production.
In the production environment 500.html is rendered whenever a server error occurs.
How do I test the rendering of 500.html in dev environment? Or how do I render 500.html in dev, if I turn-off debug I still get the errors and not 500.html
background: I include some page elements based on a page and some are missing when 500.html is called and want to debug it in dev environment.
I prefer not to turn DEBUG off. Instead I put the following snippet in the urls.py:
if settings.DEBUG:
urlpatterns += patterns('',
(r'^500/$', 'your_custom_view_if_you_wrote_one'),
(r'^404/$', 'django.views.generic.simple.direct_to_template', {'template': '404.html'}),
)
In the snippet above, the error page uses a custom view, you can easily replace it with Django's direct_to_template view though.
Now you can test 500 and 404 pages by calling their urls: http://example.com/500 and http://example.com/404
In Django 1.6 django.views.generic.simple.direct_to_template does not exists anymore, these are my settings for special views:
# urls.py
from django.views.generic import TemplateView
from django.views.defaults import page_not_found, server_error
urlpatterns += [
url(r'^400/$', TemplateView.as_view(template_name='400.html')),
url(r'^403/$', TemplateView.as_view(template_name='403.html')),
url(r'^404/$', page_not_found),
url(r'^500/$', server_error),
]
And if you want to use the default Django 500 view instead of your custom view:
if settings.DEBUG:
urlpatterns += patterns('',
(r'^500/$', 'django.views.defaults.server_error'),
(r'^404/$', 'django.views.generic.simple.direct_to_template', {'template': '404.html'}),
)
Continuing shanyu's answer, in Django 1.3+ use:
if settings.DEBUG:
urlpatterns += patterns('',
(r'^500/$', 'django.views.defaults.server_error'),
(r'^404/$', 'django.views.defaults.page_not_found'),
)
For Django > 3.0, just set the raise_request_exception value to False.
from django.test import TestCase
class ViewTestClass(TestCase):
def test_error_page(self):
self.client.raise_request_exception = False
response = self.client.get(reverse('error-page'))
self.assertEqual(response.status_code, 500)
self.assertTrue(
'some text from the custom 500 page'
in response.content.decode('utf8'))
Documentation: https://docs.djangoproject.com/en/3.2/topics/testing/tools/
NOTE: if the error page raises an exception, that will show up as an ERROR in the test log. You can turn the test logging up to CRITICAL by default to suppress that error.
Are both debug settings false?
settings.DEBUG = False
settings.TEMPLATE_DEBUG = False
How i do and test custom error handlers
Define custom View based on TemplateView
# views.py
from django.views.generic import TemplateView
class ErrorHandler(TemplateView):
""" Render error template """
error_code = 404
template_name = 'index/error.html'
def dispatch(self, request, *args, **kwargs):
""" For error on any methods return just GET """
return self.get(request, *args, **kwargs)
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['error_code'] = self.error_code
return context
def render_to_response(self, context, **response_kwargs):
""" Return correct status code """
response_kwargs = response_kwargs or {}
response_kwargs.update(status=self.error_code)
return super().render_to_response(context, **response_kwargs)
Tell django to use custom error handlers
# urls.py
from index.views import ErrorHandler
# error handing handlers - fly binding
for code in (400, 403, 404, 500):
vars()['handler{}'.format(code)] = ErrorHandler.as_view(error_code=code)
Testcase for custom error handlers
# tests.py
from unittest import mock
from django.test import TestCase
from django.core.exceptions import SuspiciousOperation, PermissionDenied
from django.http import Http404
from index import views
class ErrorHandlersTestCase(TestCase):
""" Check is correct error handlers work """
def raise_(exception):
def wrapped(*args, **kwargs):
raise exception('Test exception')
return wrapped
def test_index_page(self):
""" Should check is 200 on index page """
response = self.client.get('/')
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, 'index/index.html')
#mock.patch('index.views.IndexView.get', raise_(Http404))
def test_404_page(self):
""" Should check is 404 page correct """
response = self.client.get('/')
self.assertEqual(response.status_code, 404)
self.assertTemplateUsed(response, 'index/error.html')
self.assertIn('404 Page not found', response.content.decode('utf-8'))
#mock.patch('index.views.IndexView.get', views.ErrorHandler.as_view(error_code=500))
def test_500_page(self):
""" Should check is 500 page correct """
response = self.client.get('/')
self.assertEqual(response.status_code, 500)
self.assertTemplateUsed(response, 'index/error.html')
self.assertIn('500 Server Error', response.content.decode('utf-8'))
#mock.patch('index.views.IndexView.get', raise_(SuspiciousOperation))
def test_400_page(self):
""" Should check is 400 page correct """
response = self.client.get('/')
self.assertEqual(response.status_code, 400)
self.assertTemplateUsed(response, 'index/error.html')
self.assertIn('400 Bad request', response.content.decode('utf-8'))
#mock.patch('index.views.IndexView.get', raise_(PermissionDenied))
def test_403_page(self):
""" Should check is 403 page correct """
response = self.client.get('/')
self.assertEqual(response.status_code, 403)
self.assertTemplateUsed(response, 'index/error.html')
self.assertIn('403 Permission Denied', response.content.decode('utf-8'))
urls.py
handler500 = 'project.apps.core.views.handler500'
handler404 = 'project.apps.core.views.handler404'
views.py
from django.template.loader import get_template
from django.template import Context
from django.http import HttpResponseServerError, HttpResponseNotFound
def handler500(request, template_name='500.html'):
t = get_template(template_name)
ctx = Context({})
return HttpResponseServerError(t.render(ctx))
def handler404(request, template_name='404.html'):
t = get_template(template_name)
ctx = Context({})
return HttpResponseNotFound(t.render(ctx))
tests.py
from django.test import TestCase
from django.test.client import RequestFactory
from project import urls
from ..views import handler404, handler500
class TestErrorPages(TestCase):
def test_error_handlers(self):
self.assertTrue(urls.handler404.endswith('.handler404'))
self.assertTrue(urls.handler500.endswith('.handler500'))
factory = RequestFactory()
request = factory.get('/')
response = handler404(request)
self.assertEqual(response.status_code, 404)
self.assertIn('404 Not Found!!', unicode(response))
response = handler500(request)
self.assertEqual(response.status_code, 500)
self.assertIn('500 Internal Server Error', unicode(response))
Update for Django > 1.6 and without getting
page_not_found() missing 1 required positional argument: 'exception'
Inspired by this answer:
# urls.py
from django.views.defaults import page_not_found, server_error, permission_denied, bad_request
[...]
if settings.DEBUG:
# This allows the error pages to be debugged during development, just visit
# these url in browser to see how these error pages look like.
urlpatterns += [
path('400/', bad_request, kwargs={'exception': Exception('Bad Request!')}),
path('403/', permission_denied, kwargs={'exception': Exception('Permission Denied')}),
path('404/', page_not_found, kwargs={'exception': Exception('Page not Found')}),
path('500/', server_error),
You can simply define the handler404 and handler500 for errors in your main views.py file as detailed in this answer:
https://stackoverflow.com/a/18009660/1913888
This will return the error that you desire when Django routes to that handler. No custom URL configuration is needed to route to a different URL name.
In Django versions < 3.0, you should do as follows:
client.py
from django.core.signals import got_request_exception
from django.template import TemplateDoesNotExist
from django.test import signals
from django.test.client import Client as DjangoClient, store_rendered_templates
from django.urls import resolve
from django.utils import six
from django.utils.functional import SimpleLazyObject, curry
class Client(DjangoClient):
"""Test client that does not raise Exceptions if requested."""
def __init__(self,
enforce_csrf_checks=False,
raise_request_exception=True, **defaults):
super(Client, self).__init__(enforce_csrf_checks=enforce_csrf_checks,
**defaults)
self.raise_request_exception = raise_request_exception
def request(self, **request):
"""
The master request method. Composes the environment dictionary
and passes to the handler, returning the result of the handler.
Assumes defaults for the query environment, which can be overridden
using the arguments to the request.
"""
environ = self._base_environ(**request)
# Curry a data dictionary into an instance of the template renderer
# callback function.
data = {}
on_template_render = curry(store_rendered_templates, data)
signal_uid = "template-render-%s" % id(request)
signals.template_rendered.connect(on_template_render,
dispatch_uid=signal_uid)
# Capture exceptions created by the handler.
exception_uid = "request-exception-%s" % id(request)
got_request_exception.connect(self.store_exc_info,
dispatch_uid=exception_uid)
try:
try:
response = self.handler(environ)
except TemplateDoesNotExist as e:
# If the view raises an exception, Django will attempt to show
# the 500.html template. If that template is not available,
# we should ignore the error in favor of re-raising the
# underlying exception that caused the 500 error. Any other
# template found to be missing during view error handling
# should be reported as-is.
if e.args != ('500.html',):
raise
# Look for a signalled exception, clear the current context
# exception data, then re-raise the signalled exception.
# Also make sure that the signalled exception is cleared from
# the local cache!
response.exc_info = self.exc_info # Patch exception handling
if self.exc_info:
exc_info = self.exc_info
self.exc_info = None
if self.raise_request_exception: # Patch exception handling
six.reraise(*exc_info)
# Save the client and request that stimulated the response.
response.client = self
response.request = request
# Add any rendered template detail to the response.
response.templates = data.get("templates", [])
response.context = data.get("context")
response.json = curry(self._parse_json, response)
# Attach the ResolverMatch instance to the response
response.resolver_match = SimpleLazyObject(
lambda: resolve(request['PATH_INFO'])
)
# Flatten a single context. Not really necessary anymore thanks to
# the __getattr__ flattening in ContextList, but has some edge-case
# backwards-compatibility implications.
if response.context and len(response.context) == 1:
response.context = response.context[0]
# Update persistent cookie data.
if response.cookies:
self.cookies.update(response.cookies)
return response
finally:
signals.template_rendered.disconnect(dispatch_uid=signal_uid)
got_request_exception.disconnect(dispatch_uid=exception_uid)
tests.py
from unittest import mock
from django.contrib.auth import get_user_model
from django.core.urlresolvers import reverse
from django.test import TestCase, override_settings
from .client import Client # Important, we use our own Client here!
class TestErrors(TestCase):
"""Test errors."""
#classmethod
def setUpClass(cls):
super(TestErrors, cls).setUpClass()
cls.username = 'admin'
cls.email = 'admin#localhost'
cls.password = 'test1234test1234'
cls.not_found_url = '/i-do-not-exist/'
cls.internal_server_error_url = reverse('password_reset')
def setUp(self):
super(TestErrors, self).setUp()
User = get_user_model()
User.objects.create_user(
self.username,
self.email,
self.password,
is_staff=True,
is_active=True
)
self.client = Client(raise_request_exception=False)
# Mock in order to trigger Exception and resulting Internal server error
#mock.patch('django.contrib.auth.views.PasswordResetView.form_class', None)
#override_settings(DEBUG=False)
def test_errors(self):
self.client.login(username=self.username, password=self.password)
with self.subTest("Not found (404)"):
response = self.client.get(self.not_found_url, follow=True)
self.assertNotIn('^admin/', str(response.content))
with self.subTest("Internal server error (500)"):
response = self.client.get(self.internal_server_error_url,
follow=True)
self.assertNotIn('TypeError', str(response.content))
Starting from Django 3.0 you could skip the custom Client definition and just use the code from tests.py.