I have a django application running under twisted with the following service:
class JungoHttpService(internet.TCPServer):
def __init__(self, port):
self.__port = port
pool = threadpool.ThreadPool()
wsgi_resource = TwoStepResource(reactor, pool, WSGIHandler())
internet.TCPServer.__init__(self, port, Site(wsgi_resource))
self.setName("WSGI/HttpJungo")
self.pool = pool
def startService(self):
internet.TCPServer.startService(self)
self.pool.start()
def stopService(self):
self.pool.stop()
return internet.TCPServer.stopService(self)
def getServerPort(self):
""" returns the port number the server is listening on"""
return self.__port
Here is my TwoStepResource:
class TwoStepResource(WSGIResource):
def render (self, request):
if request.postpath:
pathInfo = '/' + '/'.join(request.postpath)
else:
pathInfo = ''
try:
callback, callback_args,
callback_kwargs = urlresolvers.resolve(pathInfo)
if hasattr(callback, "async"):
# Patch the request
_patch_request(request, callback, callback_args,
callback_kwargs)
except Exception, e:
logging.getLogger('jungo.request').error("%s : %s\n%s" % (
e.__class__.__name__, e, traceback.format_exc()))
raise
finally:
return super(TwoStepResource, self).render(request)
How can I add serving media files ("/media") to the same port?
Just add wsgi_resource.putChild('media', File("/path/to/media")) after the wsgi_resource assignment. You'll need to from twisted.web.static import File of course.
Update 1:
Turns out WSGIResource rejects putChild() attempts. There's a solution here: http://blog.vrplumber.com/index.php?/archives/2426-Making-your-Twisted-resources-a-url-sub-tree-of-your-WSGI-resource....html
Update 2:
jungo.py
from twisted.application import internet
from twisted.web import resource, wsgi, static, server
from twisted.python import threadpool
from twisted.internet import reactor
def wsgiApplication(environ, start_response):
start_response('200 OK', [('Content-type', 'text/plain')])
return ['Hello, world!']
class SharedRoot(resource.Resource):
"""Root resource that combines the two sites/entry points"""
WSGI = None
def getChild(self, child, request):
request.prepath.pop()
request.postpath.insert(0, child)
return self.WSGI
def render(self, request):
return self.WSGI.render(request)
class JungoHttpService(internet.TCPServer):
def __init__(self, port):
self.__port = port
pool = threadpool.ThreadPool()
sharedRoot = SharedRoot()
# substitute with your custom WSGIResource
sharedRoot.WSGI = wsgi.WSGIResource(reactor, pool, wsgiApplication)
sharedRoot.putChild('media', static.File("/path/to/media"))
internet.TCPServer.__init__(self, port, server.Site(sharedRoot))
self.setName("WSGI/HttpJungo")
self.pool = pool
def startService(self):
internet.TCPServer.startService(self)
self.pool.start()
def stopService(self):
self.pool.stop()
return internet.TCPServer.stopService(self)
def getServerPort(self):
""" returns the port number the server is listening on"""
return self.__port
jungo.tac
from twisted.application import internet, service
from jungo import JungoHttpService
application = service.Application("jungo")
jungoService = JungoHttpService(8080)
jungoService.setServiceParent(application)
$ twistd -n -y jungo.tac
Related
I am using flask_login, It worked in local but when i moved to different server it is did not work. On trying to debug i realized login_manager.user_loader is not getting on remote machine but the same worked in my local.
This is issue is specific to okta sso authentication on remote machine only. Post okta authentication, i am calling below to save data in session.
login_user(user)
Post this, i was expecting user_loaded to be called but that is not happening. Tried going through document but could not understand. Can you anyone help me as why user_loader is getting called in my local machine but not on remote machine.
init.py
import os
from flask import Flask
from flask_login import LoginManager
from config import app_config
login_manager = LoginManager()
def create_app(config_name):
app = Flask(__name__,
instance_path=os.path.join(os.path.abspath(os.curdir), 'instance'),
instance_relative_config=True)
app.config.from_object(app_config[config_name])
app.config.from_pyfile('config.py')
login_manager.init_app(app)
login_manager.login_message = 'You must be logged in to access this page'
login_manager.login_view = 'admin.login'
login_manager.login_message_category = 'info'
login_manager.session_protection = "strong"
#login_manager.refresh_view ='admin.login'
#login_manager.needs_refresh_message = (u"To protect your account, please reauthenticate to access this page.")
from .home import home as home_blueprint
app.register_blueprint(home_blueprint)
from .restapi import restapi as restapi_blueprint
app.register_blueprint(restapi_blueprint, url_prefix='/steps/api/v1')
return app
In my blueprint view, i am calling login_user to set current user. I have placed user_loader in one seperate file.
from flask import current_app
from flask_login import UserMixin
from app import login_manager, get_connection
from .encryption import encrypt_json, encrypt_password, decrypt_password # noqa
class User(UserMixin):
"""Custom User class."""
id = None
name = None
email = None
description = None
role = None
def __init__(self, id_, name, email, description, role):
print(name)
self.id = str(id_)
self.name = name
self.email = email
self.description = description
self.role = role
def __repr__(self):
return self.name
def claims(self):
"""Use this method to render all assigned claims on profile page."""
return {'name': self.name,
'email': self.email}.items()
#staticmethod
def get(user_id):
try:
connection = get_connection()
user_sql = "SELECT ID, USER_NAME, DESCRIPTION, EMAIL, ROLE FROM USER WHERE ID = {}"
cursor = connection.cursor()
cursor.execute(user_sql.format(user_id))
data = cursor.fetchone()
if data:
return User(id_=data[0], name=data[1], email=data[3], description=data[2], role=data[4])
else:
return None
except Exception as ex:
print(ex)
finally:
cursor.close()
connection.close()
# print(user_id)
# print(USERS_DB)
# return USERS_DB.get(user_id)
#staticmethod
def getByName(user_name):
try:
connection = get_connection()
user_sql = "SELECT ID, USER_NAME, DESCRIPTION, EMAIL, ROLE FROM USER WHERE USER_NAME = '{}' "
cursor = connection.cursor()
cursor.execute(user_sql.format(user_name))
data = cursor.fetchone()
if data:
return User(id_=data[0], name=data[1], email=data[3], description=data[2], role=data[4])
else:
return None
except Exception as ex:
print(ex)
finally:
cursor.close()
connection.close()
#staticmethod
def getByEmail(emailid):
try:
connection = get_connection()
user_sql = "SELECT ID, USER_NAME, DESCRIPTION, EMAIL, ROLE FROM USER WHERE EMAIL = '{}' "
cursor = connection.cursor(buffered=True)
cursor.execute(user_sql.format(emailid))
data = cursor.fetchone()
print(data)
if data:
return User(id_=data[0], name=data[1], email=data[3], description=data[2], role=data[4])
else:
return None
except Exception as ex:
print(ex)
finally:
cursor.close()
connection.close()
#staticmethod
def verify_password(username, password):
try:
# mysql_hook = MySqlHook(mysql_conn_id="STEPS", schema="STEPS")
# connection = mysql_hook.get_conn()
connection = get_connection()
cursor = connection.cursor()
encFlag, decryptedPassword = decrypt_password(password)
lognReqTuple = (username, password)
user_sql = "SELECT USER_NAME, PASSWORD from USER WHERE USER_NAME = '{}'"
cursor.execute(user_sql.format(username))
all_users = cursor.fetchall()
print(all_users)
for user in all_users:
encFlag, decryptedPassword = decrypt_password(user[1]) if user[1] else (False, password)
existingUserTuple = user[0], decryptedPassword
if existingUserTuple == lognReqTuple:
# logging.info('User {} authenticated.'.format(username))
return 0
# logging.info('Un Authorized access, user authentication failed.')
return 1
except Exception as ex:
# logging.error('Error in verifying login details :{}'.format(ex))
print(ex)
return 1
finally:
cursor.close()
connection.close()
#staticmethod
def create(user_id, name, email):
# USERS_DB[user_id] = User(user_id, name, email)
pass
#login_manager.user_loader
def user_loader(user_id):
print('user loader', type(user_id), user_id)
return User.get(user_id)
Problem is that, same setup is working in my local ubuntu environment but when i moved to production vm on centos, it works for one view where i am using local authentication but not in case of okta.
user_loader is how you obtain the User object. You give it an id and it gives you back the User. See the docs.
In your case you already have the User object so login_user(user) has no need to call the user_loader.
user_loader is likely called wherever you do something like user = ....
Not sure about the issue but i changed the flask app folder structure. changed the way i was using blueprints and issue got solved. All in all, had to change complete application structure.
I need to write 'one' test code with external api, which requires requests 'twice'.
first, I need to check if user is valid.
So I handled this with decorator in ./users/utils.py
import requests
def login_decorator(func):
def wrapper(self, request, *args, **kwargs):
# this access token is issued by external api
access_token = request.headers.get('Authorization', None)
# first requests, which gives me an info about user.
response = requests.get(
'https://kapi.kakao.com/v2/user/me',
headers={'Authorization':f'Bearer {access_token}'}
)
user_email = response.json()['kakao_account']['email']
request.user = User.objects.get(email=user_email)
return func(self, request, *args, **kwargs)
return wrapper
and then, I need to send that user a message with the external api again.
this code is in ./bids/views.py
import requests
class BiddingView(View):
#it uses login_decorator above
#login_decorator
def post(self, request, art_id):
try:
user = request.user
data = json.loads(request.body)
with transaction.atomic():
#handle things with bidding system#
#the external api requires its token to use a message api.
token = request.headers.get('Authorization', None)
#second requests, with post method
response = requests.post(
'https://kapi.kakao.com/v2/api/talk/memo/default/send',
headers = {'Authorization' : f'Bearer {token}'},
data = {"template_object" : json.dumps({'message':'contents'})}
)
return JsonResponse({'MESSAGE' : 'SUCCESS'}, status=200)
except KeyError:
return JsonResponse({'MESSAGE' : 'KEY ERROR'}, status=400)
This is my unit test code about BiddingView so far, which obviously only works for decorator
#patch('users.utils.requests')
def test_kakao_message_success(self, mock_requests):
class MockResponse:
def json(self):
return {'kakao_account' : {'email' : 'test#test.com'}}
mock_requests.get = MagicMock(return_value=MockResponse())
header = {'HTTP_Authorization' : 'ACCESS_TOKEN'}
body = {'offered_price' : 10000}
response = client.post(
'/bidding/1',
json.dumps(body),
content_type='application/json', **header
)
but I need to patch both .users.utils.requests and .bids.views.requests for my mock test.
#patch('users.utils.requests') # + #patch('bids.views.requests')
def test_kakao_message_success(self, mock_requests):
I want to know how to patch two requests at the same time.
I simplified your source code for ease of testing so that we can concentrate on the problem which are the external requests.
./utils.py
import requests
def login_decorator(func):
def wrapper(self, request, *args, **kwargs):
# first requests, which gives me an info about user.
response = requests.get(
'https://kapi.kakao.com/v2/user/me',
headers={'Authorization': 'Bearer access_token'}
)
request.user = response.json()['kakao_account']['email']
return func(self, request, *args, **kwargs)
return wrapper
./views.py
import json
import requests
from utils import login_decorator
class BiddingView:
#it uses login_decorator above
#login_decorator
def post(self, request, art_id):
print(f"The current user is {request.user}")
#second requests, with post method
response = requests.post(
'https://kapi.kakao.com/v2/api/talk/memo/default/send',
headers = {'Authorization' : 'Bearer token'},
data = {"template_object" : json.dumps({'message':'contents'})}
)
return response.text
Solution 1 - Manual patching of requests for each source file
You can stack the unittest.mock.patch decorator, one after the other.
from unittest.mock import MagicMock, patch
from views import BiddingView
class MockLoginResponse:
def json(self):
return {'kakao_account' : {'email' : 'test#test.com'}}
class MockViewResponse:
text = "He alone, who owns the youth, gains the future."
#patch('utils.requests.get', MagicMock(return_value=MockLoginResponse()))
#patch('views.requests.post', MagicMock(return_value=MockViewResponse()))
def test_kakao_message_success():
response = BiddingView().post(
request=MagicMock(),
art_id="some art"
)
print(f"Response: {response}")
Output:
__________________________________________________________________________________ test_kakao_message_success ___________________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
The current user is test#test.com
Response: He alone, who owns the youth, gains the future.
Solution 2.1 - Instead of patching requests per file, patch the exact target request
This requires you to install library https://pypi.org/project/requests-mock/
from unittest.mock import MagicMock
import requests_mock
from views import BiddingView
def test_kakao_message_success_with_library():
with requests_mock.Mocker() as requests_mocker:
# Mock the external requests
requests_mocker.get(
"https://kapi.kakao.com/v2/user/me",
json={'kakao_account' : {'email' : 'test#test.com'}},
)
requests_mocker.post(
"https://kapi.kakao.com/v2/api/talk/memo/default/send",
text="He alone, who owns the youth, gains the future.",
)
response = BiddingView().post(
request=MagicMock(),
art_id="some art"
)
print(f"Response: {response}")
Output:
Same as above
Solution 2.2 - Instead of patching requests per file, patch the exact target request. But now, apply it automatically to any test using pytest's autouse feature.
from unittest.mock import MagicMock
import pytest
import requests_mock
from views import BiddingView
#pytest.fixture(autouse=True)
def setup_external_requests():
with requests_mock.Mocker() as requests_mocker:
# Mock the external requests
requests_mocker.get(
"https://kapi.kakao.com/v2/user/me",
json={'kakao_account' : {'email' : 'test#test.com'}},
)
requests_mocker.post(
"https://kapi.kakao.com/v2/api/talk/memo/default/send",
text="He alone, who owns the youth, gains the future.",
)
# It is required to perform a yield instead of return to not teardown the requests_mocker
yield requests_mocker
def test_kakao_message_success_with_library_2():
response = BiddingView().post(
request=MagicMock(),
art_id="some art"
)
print(f"Response: {response}")
Output:
Same as above
I am attempting to build a server that takes user requests for long(ish)-running jobs, updates the user as the job progresses, and returns some data for the client to use. I am attempting to use tornado's WebSocketHandler to do this. Is there a reason I can't call a WebSocketHandler's write_message method from another object?
import tornado.ioloop
import tornado.websocket
import json, sys, os
from uuid import uuid4
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write('Welcome to the site. Requests cannot be made to the main page.')
class WSInvalidRequest(Exception):
"""Called when user sends invalid request to the server."""
pass
class WSRequestQueue:
def __init__(self):
self._items = []
def put(self, item):
self._items.append(item)
return self._items.length()
def get(self):
return self._items.pop(0)
def get_position(self, item):
return self._items.index(item)
QUEUE = WSRequestQueue()
class WSRequest:
def __init__(self, message, websocket):
self.websocket = websocket
self.ran = False
self.valid = False
self.write(u'Request received.')
try:
self.request = WSRequest.parse_message(message)
self.valid = True
self.write(u'Request validated.')
position = QUEUE.put(self)
self.write(u'Added request to queue behind %i other requests.' % position)
except WSInvalidRequest as e: self.write(e.message)
#staticmethod
def validate_request_dict(request):
if not isinstance(messageDict, dict):
raise WSInvalidRequest(u'Invalid request. Should be JSON dict string.')
if 'arg' not in request:
raise WSInvalidRequest(u'Invalid request. No arg found')
#staticmethod
def parse_message(message):
messageDict = json.loads(message)
validate_request_dict(messageDict)
argument = messsageDict['arg']
return {'argumet': argument}
def write(self, message):
self.websocket.write_messsage(unicode(message))
def run(self):
self.ran = True
def destroy(self):
if self.valid:
if not self.ran: QUEUE.pop(QUEUE.get_position(self))
self.websocket.requests.remove(self)
self.write(u'Removed request from queue.')
class RequestWebSocket(tornado.websocket.WebSocketHandler):
def open(self):
self.id = uuid4()
self.requests = set()
print("WebSocket opened")
def on_message(self, message):
self.write_message(u'You sent: %s' % message)
self.write_message(u'Attempting to add your request to the queue.')
newRequest = WSRequest(message, self)
if newRequest.valid: self.requests.add(newRequest)
else: newRequest.destroy
def on_close(self):
print("WebSocket closed. Removing all requests from the queue.")
for request in self.requests: request.destroy()
def check_origin(self, origin):
return True
if __name__ == "__main__":
# Create the web server
application = tornado.web.Application([
(r'/', MainHandler),
(r'/websocket', RequestWebSocket)
], debug=True)
application.listen(80)
tornado.ioloop.IOLoop.instance().start()
There's a spelling mistake in write_messsage. There's an extra s.
It's at:
class WSRequest:
def write(self, message):
self.websocket.write_messsage(unicode(message))
# ^ extra 's'
My method parse_adf_info never is called and I dont know why. No error occurs. I want to get the links for each ads (parse) and go to ads one by one (parse_ads_urls) and scraping data (parse_ads_info), but this method never is called.
Here is my code:
# -*- coding: utf-8 -*-
from scrapy import Request, Spider
#from zapimoveis.items import ads_info
from scrapy.selector import Selector
#from scrapy.loader import ItemLoader
proxy_list = ["###","###"]
PROXY = "###"
class AdsSpider(Spider):
name = "zapimoveis"
allowed_domains = ["https://www.zapimoveis.com.br/", "https://www.zapimoveis.com.br/oferta/"]
def __init__(self, start_url='', *args, **kwargs):
super(AdsSpider, self).__init__(*args, **kwargs)
self.start_urls = []
self.start_urls.append(start_url)
self.json = '#{"precomaximo":"2147483647","parametrosautosuggest":[{"B\
airro":"JD CAMBURI","Zona":"","Cidade":"VITORIA","Agrupame\
nto":"","Estado":"ES"}],"pagina":"%d","ordem":"DataAtualiz\
acao","paginaOrigem":"ResultadoBusca","semente":"213739135\
0","formato":"Lista"}'
def start_requests(self):
rq = Request(url=self.start_urls[0], callback=self.parse)
rq.meta['proxy'] = PROXY
yield rq
def parse(self, response):
n_pages = response.css('span[class="pull-right num-of"]::text') \
.extract_first()
n_pages = int(n_pages.replace("de ", ""))
for i in range(1, n_pages+1):
rq = Request(url=self.start_urls[0]+(self.json % i),
callback=self.parse_ads_urls, dont_filter=True)
rq.meta['proxy'] = PROXY
yield rq
def parse_ads_urls(self,response):
for article in response.css('article[class=minificha]'):
url_to_ads = article.css('a[class=btn-ver-detalhes]::attr(href)')\
.extract_first()
rq2 = Request(url=url_to_ads, callback=self.parse_ads_info,
dont_filter=True)
rq2.meta['proxy'] = proxy_list[0]
yield rq2
def parse_ads_info(self, response):
print "#--------->"
print response.css('span[class=value-ficha]::text').extract_first()
I removed my personal proxys.
(2017-06-06) EDIT 1:
Output log : https://pastebin.com/4jv2r9um
I am trying to learn twisted library from its documentation. I've created a simple chat server that allows users to public chat. However, I wonder if I can add private chat functionality to that. I am looking for your ideas. Thanks.
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
from twisted.internet import reactor
class Chat(LineReceiver):
def __init__(self,users):
self.users = users
self.name = None
self.state = "GETNAME"
def connectionMade(self):
self.sendLine("What's your name?")
def connectionLost(self, reason):
if self.users.has_key(self.name):
del self.users[self.name]
def lineReceived(self,line):
if self.state == "GETNAME":
self.handle_GETNAME(line)
else:
self.handle_CHAT(line)
def handle_GETNAME(self,name):
if self.users.has_key(name):
self.sendLine("Name taken, please choose another.")
return
self.sendLine("Welcome, %s!" % (name))
self.name = name
self.users[name] = self
self.state = "CHAT"
def handle_CHAT(self,msg):
msg = "<%s> %s" % (self.name,msg)
for name,protocol in self.users.iteritems():
if protocol != self:
protocol.sendLine(msg)
class ChatFactory(Factory):
def __init__(self):
self.users = {}
def buildProtocol(self, addr):
return Chat(self.users)
reactor.listenTCP(8123,ChatFactory()) ##UndefinedVariable
reactor.run() ##UndefinedVariable
Actually I figured it out by changing the for loop in handle_CHAT. As I want the program to send the message to the specific user, a "talkwith" string can be added to the class and this string can hold that specific user's name. Then this change turns the program from public to private chatting.
msg = "<%s> %s" % (self.name,msg)
for name,protocol in self.users.iteritems():
if name == self.talkwith:
protocol.sendLine(msg)