I am attempting to build a server that takes user requests for long(ish)-running jobs, updates the user as the job progresses, and returns some data for the client to use. I am attempting to use tornado's WebSocketHandler to do this. Is there a reason I can't call a WebSocketHandler's write_message method from another object?
import tornado.ioloop
import tornado.websocket
import json, sys, os
from uuid import uuid4
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write('Welcome to the site. Requests cannot be made to the main page.')
class WSInvalidRequest(Exception):
"""Called when user sends invalid request to the server."""
pass
class WSRequestQueue:
def __init__(self):
self._items = []
def put(self, item):
self._items.append(item)
return self._items.length()
def get(self):
return self._items.pop(0)
def get_position(self, item):
return self._items.index(item)
QUEUE = WSRequestQueue()
class WSRequest:
def __init__(self, message, websocket):
self.websocket = websocket
self.ran = False
self.valid = False
self.write(u'Request received.')
try:
self.request = WSRequest.parse_message(message)
self.valid = True
self.write(u'Request validated.')
position = QUEUE.put(self)
self.write(u'Added request to queue behind %i other requests.' % position)
except WSInvalidRequest as e: self.write(e.message)
#staticmethod
def validate_request_dict(request):
if not isinstance(messageDict, dict):
raise WSInvalidRequest(u'Invalid request. Should be JSON dict string.')
if 'arg' not in request:
raise WSInvalidRequest(u'Invalid request. No arg found')
#staticmethod
def parse_message(message):
messageDict = json.loads(message)
validate_request_dict(messageDict)
argument = messsageDict['arg']
return {'argumet': argument}
def write(self, message):
self.websocket.write_messsage(unicode(message))
def run(self):
self.ran = True
def destroy(self):
if self.valid:
if not self.ran: QUEUE.pop(QUEUE.get_position(self))
self.websocket.requests.remove(self)
self.write(u'Removed request from queue.')
class RequestWebSocket(tornado.websocket.WebSocketHandler):
def open(self):
self.id = uuid4()
self.requests = set()
print("WebSocket opened")
def on_message(self, message):
self.write_message(u'You sent: %s' % message)
self.write_message(u'Attempting to add your request to the queue.')
newRequest = WSRequest(message, self)
if newRequest.valid: self.requests.add(newRequest)
else: newRequest.destroy
def on_close(self):
print("WebSocket closed. Removing all requests from the queue.")
for request in self.requests: request.destroy()
def check_origin(self, origin):
return True
if __name__ == "__main__":
# Create the web server
application = tornado.web.Application([
(r'/', MainHandler),
(r'/websocket', RequestWebSocket)
], debug=True)
application.listen(80)
tornado.ioloop.IOLoop.instance().start()
There's a spelling mistake in write_messsage. There's an extra s.
It's at:
class WSRequest:
def write(self, message):
self.websocket.write_messsage(unicode(message))
# ^ extra 's'
Related
How can I prevent Django rest throttling count the request when the user request is invalid or the server failed to complete the process?
For example, I need params from the user, but when the user does not give the params, Django rest throttling still counts it.
Is there any solution to skipping the throttling counter when the request is not successful?
Example
class OncePerHourAnonThrottle(AnonRateThrottle):
rate = "1/hour"
class Autoliker(APIView):
throttle_classes = [OncePerHourAnonThrottle]
def get(self, request):
content = {"status": "get"}
return Response(content)
def post(self, request):
post_url = request.POST.get("url", None)
print(post_url)
content = {"status": "post"}
return Response(content)
def throttled(self, request, wait):
raise Throttled(
detail={
"message": "request limit exceeded",
"availableIn": f"{wait} seconds",
"throttleType": "type",
}
)
You can create a decorator to do so.
class OncePerHourAnonThrottle(AnonRateThrottle):
rate = "1/hour"
def allow_request(self, request, view):
"""
This function is copy of SimpleRateThrottle.allow_request
The only difference is, instead of executing self.throttle_success
it directly returns True and doesn't mark this request as success yet.
"""
if self.rate is None:
return True
self.key = self.get_cache_key(request, view)
if self.key is None:
return True
self.history = self.cache.get(self.key, [])
self.now = self.timer()
# Drop any requests from the history which have now passed the
# throttle duration
while self.history and self.history[-1] <= self.now - self.duration:
self.history.pop()
if len(self.history) >= self.num_requests:
return False
return True
def rate_limiter(view_function):
#wraps(view_function)
def inner(view_obj, request, *args, **kwargs):
throttle = OncePerHourAnonThrottle()
allowed = throttle.allow_request(request, None)
if not allowed:
raise exceptions.Throttled(throttle.wait())
try:
response = view_function(view_obj, request, *args, **kwargs)
except Exception as exc:
response = view_obj.handle_exception(exc)
if response.status_code == 200:
# now if everything goes OK, count this request as success
throttle.throttle_success()
return response
return inner
class Autoliker(APIView):
#rate_limiter
def post(requests):
# view logic
pass
This is the basic idea how you can do it, now you can make it a generic decorator or even class based decorator.
My method parse_adf_info never is called and I dont know why. No error occurs. I want to get the links for each ads (parse) and go to ads one by one (parse_ads_urls) and scraping data (parse_ads_info), but this method never is called.
Here is my code:
# -*- coding: utf-8 -*-
from scrapy import Request, Spider
#from zapimoveis.items import ads_info
from scrapy.selector import Selector
#from scrapy.loader import ItemLoader
proxy_list = ["###","###"]
PROXY = "###"
class AdsSpider(Spider):
name = "zapimoveis"
allowed_domains = ["https://www.zapimoveis.com.br/", "https://www.zapimoveis.com.br/oferta/"]
def __init__(self, start_url='', *args, **kwargs):
super(AdsSpider, self).__init__(*args, **kwargs)
self.start_urls = []
self.start_urls.append(start_url)
self.json = '#{"precomaximo":"2147483647","parametrosautosuggest":[{"B\
airro":"JD CAMBURI","Zona":"","Cidade":"VITORIA","Agrupame\
nto":"","Estado":"ES"}],"pagina":"%d","ordem":"DataAtualiz\
acao","paginaOrigem":"ResultadoBusca","semente":"213739135\
0","formato":"Lista"}'
def start_requests(self):
rq = Request(url=self.start_urls[0], callback=self.parse)
rq.meta['proxy'] = PROXY
yield rq
def parse(self, response):
n_pages = response.css('span[class="pull-right num-of"]::text') \
.extract_first()
n_pages = int(n_pages.replace("de ", ""))
for i in range(1, n_pages+1):
rq = Request(url=self.start_urls[0]+(self.json % i),
callback=self.parse_ads_urls, dont_filter=True)
rq.meta['proxy'] = PROXY
yield rq
def parse_ads_urls(self,response):
for article in response.css('article[class=minificha]'):
url_to_ads = article.css('a[class=btn-ver-detalhes]::attr(href)')\
.extract_first()
rq2 = Request(url=url_to_ads, callback=self.parse_ads_info,
dont_filter=True)
rq2.meta['proxy'] = proxy_list[0]
yield rq2
def parse_ads_info(self, response):
print "#--------->"
print response.css('span[class=value-ficha]::text').extract_first()
I removed my personal proxys.
(2017-06-06) EDIT 1:
Output log : https://pastebin.com/4jv2r9um
I am new to testing and need some help here.
Assuming having this method:
from urllib.request import urlopen
def get_posts():
with urlopen('some url here') as data:
return json.loads(data.read().decode('utf-8'))
The question is how to test this method (using mock.patch decorator if possible)?
What I have now:
#mock.patch('mymodule.urlopen')
def test_get_post(self, mocked_urlopen):
mocked_urlopen.__enter__ = Mock(return_value=self.test_data)
mocked_urlopen.__exit__ = Mock(return_value=False)
...
But it does not seem to be working.
P.S. Is there any convenient way to work with data variable (which type is HTTPResponse) in test so it could just be simple string?
I was fighting with this as well, and finally figured it out. (Python 3 syntax):
import urllib.request
import unittest
from unittest.mock import patch, MagicMock
class TestUrlopen(unittest.TestCase):
#patch('urllib.request.urlopen')
def test_cm(self, mock_urlopen):
cm = MagicMock()
cm.getcode.return_value = 200
cm.read.return_value = 'contents'
cm.__enter__.return_value = cm
mock_urlopen.return_value = cm
with urllib.request.urlopen('http://foo') as response:
self.assertEqual(response.getcode(), 200)
self.assertEqual(response.read(), 'contents')
#patch('urllib.request.urlopen')
def test_no_cm(self, mock_urlopen):
cm = MagicMock()
cm.getcode.return_value = 200
cm.read.return_value = 'contents'
mock_urlopen.return_value = cm
response = urllib.request.urlopen('http://foo')
self.assertEqual(response.getcode(), 200)
self.assertEqual(response.read(), 'contents')
response.close()
here is my take on this
from urllib.request import urlopen
from unittest.mock import patch
class Mock():
def __init__(self, request, context):
return None
def read(self):
return self
def decode(self, arg):
return ''
def __iter__(self):
return self
def __next__(self):
raise StopIteration
with patch('urllib.request.urlopen', Mock):
# do whatever over here
with urlopen('some url here') as data is a context manager
Also, a file can be used as a context manager, so a better approach here is to use io.StringIO
import io
import json
import urllib.request
from unittest.mock import patch
def get_posts():
with urllib.request.urlopen('some url here') as data:
return json.load(data)
def test_get_posts():
data = io.StringIO('{"id": 123}')
with patch.object(urllib.request, 'urlopen', return_value=data):
assert get_posts() == {"id": 123}
Ok, so I have written simple class to simulate context manager.
class PatchContextManager:
def __init__(self, method, enter_return, exit_return=False):
self._patched = patch(method)
self._enter_return = enter_return
self._exit_return = exit_return
def __enter__(self):
res = self._patched.__enter__()
res.context = MagicMock()
res.context.__enter__.return_value = self._enter_return
res.context.__exit__.return_value = self._exit_return
res.return_value = res.context
return res
def __exit__(self, type, value, tb):
return self._patched.__exit__()
Usage:
with PatchContextManager('mymodule.method', 'return_string') as mocked:
a = mymodule.method(47) # a == 'return_string'
mocked.assert_called_with(47)
...
I recently ask this question Custom Django MultilingualTextField model field but I found no good reason why I should not do this, so I create a model Field that support multilingual text, auto return text in current language. This basically is the field that store custom Language object to database in json format. Here is the code:
Github: https://github.com/james4388/django-multilingualfield
Ussage:
from django.db import models
from multilingualfield import MLTextField, MLHTMLField
class MyModel(models.Model):
text = MLTextField()
html = MLHTMLField()
Used it like normal text field, translation is auto bases on system language (translation.get_language)
>>>from django.utils import translation
>>>translation.active('en')
>>>m = MyModal.objects.create(text='Hello world',html='<b>Hello world</b>');
>>>m.text
Hello world
>>>translation.active('fr')
>>>m.text #Auto fallback to first language (if any).
Hello world
>>>m.text.value('Bonjour')
>>>m.text.value('Ciao','es')
>>>m.text
Bonjour
>>>m.save()
>>>m.text.get_available_language()
['en', 'fr', 'es']
>>>m.text.remove_language('en')
Field.py
from __future__ import unicode_literals
from django.core.exceptions import ValidationError
from django.conf import settings
from django.db import models, DatabaseError, transaction
from django.utils.translation import ugettext_lazy as _, get_language
from django.utils import six
try:
import json
except ImportError:
from django.utils import simplejson as json
def get_base_language(lang):
if '-' in lang:
return lang.split('-')[0]
return lang
def get_current_language(base=True):
l = get_language()
if base:
return get_base_language(l)
return l
from .widgets import MultilingualWidget, MultilingualHTMLWidget
from .forms import MultilingualTextFormField, MultilingualHTMLFormField
from .language import LanguageText
class MultilingualTextField(six.with_metaclass(models.SubfieldBase, models.Field)):
"""
A field that support multilingual text for your model
"""
default_error_messages = {
'invalid': _("'%s' is not a valid JSON string.")
}
description = "Multilingual text field"
def __init__(self, *args, **kwargs):
self.lt_max_length = kwargs.pop('max_length',-1)
self.default_language = kwargs.get('default_language', get_current_language())
super(MultilingualTextField, self).__init__(*args, **kwargs)
def formfield(self, **kwargs):
defaults = {
'form_class': MultilingualTextFormField,
'widget': MultilingualWidget
}
defaults.update(**kwargs)
return super(MultilingualTextField, self).formfield(**defaults)
def validate(self, value, model_instance):
if not self.null and value is None:
raise ValidationError(self.error_messages['null'])
try:
self.get_prep_value(value)
except:
raise ValidationError(self.error_messages['invalid'] % value)
def get_internal_type(self):
return 'TextField'
def db_type(self, connection):
return 'text'
def to_python(self, value):
if isinstance(value, six.string_types):
if value == "" or value is None:
if self.null:
return None
if self.blank:
return ""
try:
valuejson = json.loads(value)
Lang = LanguageText(max_length=self.lt_max_length,default_language=self.default_language)
Lang.values = valuejson
return Lang
except ValueError:
try:
Lang = LanguageText(value,language=None,max_length=self.lt_max_length,default_language=self.default_language)
return Lang
except:
msg = self.error_messages['invalid'] % value
raise ValidationError(msg)
return value
def get_db_prep_value(self, value, connection=None, prepared=None):
return self.get_prep_value(value)
def get_prep_value(self, value):
if value is None:
if not self.null and self.blank:
return ""
return None
if isinstance(value, six.string_types):
value = LanguageText(value,language=None,max_length=self.lt_max_length,default_language=self.default_language)
if isinstance(value, LanguageText):
value.max_length = self.lt_max_length
value.default_language = self.default_language
return json.dumps(value.values)
return None
def get_prep_lookup(self, lookup_type, value):
if lookup_type in ["exact", "iexact"]:
return self.to_python(self.get_prep_value(value))
if lookup_type == "in":
return [self.to_python(self.get_prep_value(v)) for v in value]
if lookup_type == "isnull":
return value
if lookup_type in ["contains", "icontains"]:
if isinstance(value, (list, tuple)):
raise TypeError("Lookup type %r not supported with argument of %s" % (
lookup_type, type(value).__name__
))
# Need a way co combine the values with '%', but don't escape that.
return self.get_prep_value(value)[1:-1].replace(', ', r'%')
if isinstance(value, dict):
return self.get_prep_value(value)[1:-1]
return self.to_python(self.get_prep_value(value))
raise TypeError('Lookup type %r not supported' % lookup_type)
def value_to_string(self, obj):
return self._get_val_from_obj(obj)
Forms.py
from django import forms
from django.utils import simplejson as json
from .widgets import MultilingualWidget, MultilingualHTMLWidget
from .language import LanguageText
class MultilingualTextFormField(forms.CharField):
widget = MultilingualWidget
def __init__(self, *args, **kwargs):
kwargs['widget'] = MultilingualWidget
super(MultilingualTextFormField, self).__init__(*args, **kwargs)
def clean(self, value):
"""
The default is to have a TextField, and we will decode the string
that comes back from this. However, another use of this field is
to store a list of values, and use these in a MultipleSelect
widget. So, if we have an object that isn't a string, then for now
we will assume that is where it has come from.
"""
value = super(MultilingualTextFormField, self).clean(value)
if not value:
return value
if isinstance(value, basestring):
try:
valuejson = json.loads(value)
Lang = LanguageText()
Lang.values = valuejson
return Lang
except ValueError:
try:
Lang = LanguageText(value,language=None)
return Lang
except:
raise forms.ValidationError(
'JSON decode error: %s' % (unicode(exc),)
)
else:
return value
Language object in language.py
from __future__ import unicode_literals
from django.core.exceptions import ValidationError
from django.conf import settings
from django.db import models, DatabaseError, transaction
from django.utils.translation import ugettext_lazy as _, get_language
try:
import json
except ImportError:
from django.utils import simplejson as json
def get_base_language(lang):
if '-' in lang:
return lang.split('-')[0]
return lang
def get_current_language(base=True):
l = get_language()
if base:
return get_base_language(l)
return l
class LanguageText(object):
'''
JSON text field blah blah blah
'''
values = {}
default_language = None
max_length = -1
def __init__(self, value=None, language=None, default_language=None, max_length=-1):
self.max_length = max_length
self.default_language = default_language
self.values = {}
if value is not None:
self.value(value,language)
def __call__(self, value=None, language=None):
self.value(value,language)
return self
def get_available_language(self):
return self.values.keys()
def get_current_language(self, base=False):
return get_current_language(base)
def remove_language(self, lang):
try:
return self.values.pop(lang)
except:
pass
def has_language(self, lang):
return self.values.has_key(lang)
def get(self, language=None, fallback=True):
if language is None:
curr_lang = get_current_language(False)
else:
curr_lang = language
curr_lang_base = get_current_language(True)
if curr_lang in self.values:
return self.values[curr_lang]
if not fallback:
return None
if curr_lang_base in self.values:
return self.values[curr_lang_base]
if self.default_language in self.values:
return self.values[self.default_language]
try:
first_lang = self.values.keys()[0]
return self.values[first_lang]
except:
pass
return None
def value(self, value=None, language=None):
if value is None: #Get value
return self.get(language)
else: #Set value
if language is None:
language = get_current_language(False)
if self.max_length != -1:
value = value[:self.max_length]
self.values[language] = value
return None
def __unicode__(self):
return self.value()
def __str__(self):
return unicode(self.value()).encode('utf-8')
def __repr__(self):
return unicode(self.value()).encode('utf-8')
widgets.py
from django import forms
from django.utils import simplejson as json
from django.conf import settings
from .language import LanguageText
from django.template import loader, Context
class MultilingualWidget(forms.Textarea):
def __init__(self, *args, **kwargs):
forms.Widget.__init__(self, *args, **kwargs)
def render(self, name, value, attrs=None):
if value is None: #New create or edit none
vjson = '{}'
aLang = []
Lang = '[]'
Langs = json.dumps(dict(settings.LANGUAGES))
t = loader.get_template('multilingualtextarea.html')
c = Context({"data":value,"vjson":vjson,"lang":Lang,"langs":Langs,"langobjs":settings.LANGUAGES,"fieldname":name})
return t.render(c)
if isinstance(value, LanguageText):
vjson = json.dumps(value.values)
aLang = value.get_available_language()
Lang = json.dumps(aLang)
Langs = json.dumps(dict(settings.LANGUAGES))
t = loader.get_template('multilingualtextarea.html')
c = Context({"data":value,"vjson":vjson,"lang":Lang,"langs":Langs,"langobjs":settings.LANGUAGES,"fieldname":name})
return t.render(c)
return "Invalid data '%s'" % value
So I would like to know is this a good approach? Why shouldn't I do this? Plz help
Code looks good to me.
The only thing that could impact performance is the frequent json encoding/decoding... yet, that shouldn't have a major impact unless you are facing thousands of users on a server with minimal resources.
The previous question you linked to contains some comments noting that adding additional languages might be easier using other means. But in the end - that's a mixture between personal preferences and maintainability. If it fits your project goals, I can't see any reason not to do it the way you've coded it.
Providing proof that your implementation is the best is near to impossible. That is, unless you prove it yourself by creating a different, non-json based implementation and benchmark both on your production server. You'll notice differences will be rather minimal on regular machines. Yet, only the individual numbers will provide actual proof and can help you decide if it's "tuned" and "resource -friendly" enough for your project's purposes. I think it will fit your needs... but that's only my 2 cents.
I have a django application running under twisted with the following service:
class JungoHttpService(internet.TCPServer):
def __init__(self, port):
self.__port = port
pool = threadpool.ThreadPool()
wsgi_resource = TwoStepResource(reactor, pool, WSGIHandler())
internet.TCPServer.__init__(self, port, Site(wsgi_resource))
self.setName("WSGI/HttpJungo")
self.pool = pool
def startService(self):
internet.TCPServer.startService(self)
self.pool.start()
def stopService(self):
self.pool.stop()
return internet.TCPServer.stopService(self)
def getServerPort(self):
""" returns the port number the server is listening on"""
return self.__port
Here is my TwoStepResource:
class TwoStepResource(WSGIResource):
def render (self, request):
if request.postpath:
pathInfo = '/' + '/'.join(request.postpath)
else:
pathInfo = ''
try:
callback, callback_args,
callback_kwargs = urlresolvers.resolve(pathInfo)
if hasattr(callback, "async"):
# Patch the request
_patch_request(request, callback, callback_args,
callback_kwargs)
except Exception, e:
logging.getLogger('jungo.request').error("%s : %s\n%s" % (
e.__class__.__name__, e, traceback.format_exc()))
raise
finally:
return super(TwoStepResource, self).render(request)
How can I add serving media files ("/media") to the same port?
Just add wsgi_resource.putChild('media', File("/path/to/media")) after the wsgi_resource assignment. You'll need to from twisted.web.static import File of course.
Update 1:
Turns out WSGIResource rejects putChild() attempts. There's a solution here: http://blog.vrplumber.com/index.php?/archives/2426-Making-your-Twisted-resources-a-url-sub-tree-of-your-WSGI-resource....html
Update 2:
jungo.py
from twisted.application import internet
from twisted.web import resource, wsgi, static, server
from twisted.python import threadpool
from twisted.internet import reactor
def wsgiApplication(environ, start_response):
start_response('200 OK', [('Content-type', 'text/plain')])
return ['Hello, world!']
class SharedRoot(resource.Resource):
"""Root resource that combines the two sites/entry points"""
WSGI = None
def getChild(self, child, request):
request.prepath.pop()
request.postpath.insert(0, child)
return self.WSGI
def render(self, request):
return self.WSGI.render(request)
class JungoHttpService(internet.TCPServer):
def __init__(self, port):
self.__port = port
pool = threadpool.ThreadPool()
sharedRoot = SharedRoot()
# substitute with your custom WSGIResource
sharedRoot.WSGI = wsgi.WSGIResource(reactor, pool, wsgiApplication)
sharedRoot.putChild('media', static.File("/path/to/media"))
internet.TCPServer.__init__(self, port, server.Site(sharedRoot))
self.setName("WSGI/HttpJungo")
self.pool = pool
def startService(self):
internet.TCPServer.startService(self)
self.pool.start()
def stopService(self):
self.pool.stop()
return internet.TCPServer.stopService(self)
def getServerPort(self):
""" returns the port number the server is listening on"""
return self.__port
jungo.tac
from twisted.application import internet, service
from jungo import JungoHttpService
application = service.Application("jungo")
jungoService = JungoHttpService(8080)
jungoService.setServiceParent(application)
$ twistd -n -y jungo.tac