I am trying to deploy a simple standard app engine in python and from there to make bigquery queries through python bigquery client.
The code is as simple as these:
from __future__ import absolute_import
import webapp2
import os
from google.cloud import bigquery
class MainPage(webapp2.RequestHandler):
def get(self):
client = bigquery.Client(project = "ancient-ceiling-125223")
project_name = str(client.project)
query_job = client.query("select 1")
assert query_job.state == 'RUNNING'
iterator = query_job.result(timeout= 30)
rows = list(iterator)
self.response.write('nothing to see %s' % (project_name))
app = webapp2.WSGIApplication(
[('/', MainPage)], debug=True)
The error log: The error appears in the dummy query request
*('Connection broken: IncompleteRead(209 bytes read)', IncompleteRead(209 bytes read)) (/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py:1528)
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/main.py", line 35, in get
query_job = client.query("select 1")
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/cloud/bigquery/client.py", line 986, in query
job._begin(retry=retry)
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/cloud/bigquery/job.py", line 397, in _begin
method='POST', path=path, data=self._build_resource())
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/cloud/bigquery/client.py", line 271, in _call_api
return call()
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/api_core/retry.py", line 260, in retry_wrapped_func
on_error=on_error,
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/api_core/retry.py", line 177, in retry_target
return target()
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/cloud/_http.py", line 290, in api_request
headers=headers, target_object=_target_object)
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/cloud/_http.py", line 183, in _make_request
return self._do_request(method, url, headers, data, target_object)
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/cloud/_http.py", line 212, in _do_request
url=url, method=method, headers=headers, data=data)
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/google/auth/transport/requests.py", line 186, in request
method, url, data=data, headers=request_headers, **kwargs)
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/requests/sessions.py", line 502, in request
resp = self.send(prep, **send_kwargs)
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/requests/sessions.py", line 652, in send
r.content
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/requests/models.py", line 825, in content
self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
File "/base/data/home/apps/s~ancient-ceiling-125223/20171115t104156.405543624689752939/lib/requests/models.py", line 750, in generate
raise ChunkedEncodingError(e)
ChunkedEncodingError: ('Connection broken: IncompleteRead(209 bytes read)', IncompleteRead(209 bytes read))*
I could not make it work using python bigquery client library, Here is what I found working in standard app engine environment,
from __future__ import absolute_import
import webapp2
from googleapiclient.discovery import build
from oauth2client.client import GoogleCredentials
class MainPage(webapp2.RequestHandler):
def get(self):
credentials = GoogleCredentials.get_application_default()
service = build('bigquery', 'v2', credentials=credentials)
datasets = service.datasets().list(projectId="ancient-ceiling-125223").execute()
self.response.write('datasets: %s' % datasets)
app = webapp2.WSGIApplication(
[('/', MainPage)], debug=True)
Related
I'm attempting to write a unit test for a url in my application. I used django's Client class to simulate a get() request and compare the response's status code.
Here's the test i'm running:
from unittest.mock import patch
from django.shortcuts import reverse
class DashboardViewTest(TestCase):
#patch("ordering.mixins.OrderingAppPermissionRequired.handle_not_logged_in")
#patch("ordering.mixins.OrderingAppPermissionRequired.handle_no_profile")
#patch("ordering.mixins.OrderingAppPermissionRequired.handle_no_id")
def test_order_list_view(self, *mocks):
client = Client()
response = client.get(reverse('ordering:list'))
self.assertEqual(response.status_code, 200)
I'm facing the following error (path redacted for privacy):
Traceback (most recent call last):
File "[python_root]\python\python37\Lib\unittest\mock.py", line 1191, in patched
return func(*args, **keywargs)
File "[project_root]\ordering\tests\test_dashboard.py", line 20, in test_order_list_view
response = client.get(reverse('ordering:list'))
File "[virtual_env_root]\lib\site-packages\django\test\client.py", line 527, in get
response = super().get(path, data=data, secure=secure, **extra)
File "[virtual_env_root]\lib\site-packages\django\test\client.py", line 339, in get
**extra,
File "[virtual_env_root]\lib\site-packages\django\test\client.py", line 414, in generic
return self.request(**r)
File "[virtual_env_root]\lib\site-packages\django\test\client.py", line 495, in request
raise exc_value
File "[virtual_env_root]\lib\site-packages\django\core\handlers\exception.py", line 34, in inner
response = get_response(request)
File "[virtual_env_root]\lib\site-packages\django\utils\deprecation.py", line 93, in __call__
response = self.process_response(request, response)
File "[virtual_env_root]\lib\site-packages\django\contrib\sessions\middleware.py", line 45, in process_response
patch_vary_headers(response, ('Cookie',))
File "[virtual_env_root]\lib\site-packages\django\utils\cache.py", line 266, in patch_vary_headers
vary_headers = cc_delim_re.split(response['Vary'])
TypeError: expected string or bytes-like object
Help is appreciated. Thank you.
I'm trying to deploy my telegram-bot on PythonAnywhere. It has worked fine with the free account but there are some troubles with the paid one. I'm getting "OpenSSL.SSL.Error: [('SSL routines', 'ssl3_get_record', 'decryption failed or bad record mac')]" messages in Error logs (accordingly bot works wrong). How can I solve it?
I'm using python3.7 + pyTelegramBotAPI + flask
Code example for error reproducing:
# coding=utf-8
import telebot
import flask
import time
token = 'bot_token'
bot = telebot.TeleBot(token, threaded=False)
WEBHOOK_HOST = '*userName*.pythonanywhere.com'
WEBHOOK_URL_BASE = "https://%s" % (WEBHOOK_HOST)
WEBHOOK_URL_PATH = "/%s/" % (token)
app = flask.Flask(__name__)
# Process webhook calls
#app.route(WEBHOOK_URL_PATH, methods=['POST'])
def webhook():
if flask.request.headers.get('content-type') == 'application/json':
json_string = flask.request.get_data().decode('utf-8')
update = telebot.types.Update.de_json(json_string)
bot.process_new_updates([update])
return ''
else:
flask.abort(403)
#bot.message_handler(commands=['start', 'help'])
def handle_start_help(message):
bot.send_message(message.chat.id, text="Hello, my friend")
bot.remove_webhook()
time.sleep(0.1)
bot.set_webhook(url=WEBHOOK_URL_BASE + WEBHOOK_URL_PATH)
Full error trace:
2018-10-30 12:40:33,351: Exception on /*token*/ [POST]
Traceback (most recent call last):
File "/usr/lib/python3.7/site-packages/flask/app.py", line 2292, in wsgi_app
response = self.full_dispatch_request()
File "/usr/lib/python3.7/site-packages/flask/app.py", line 1815, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/lib/python3.7/site-packages/flask/app.py", line 1718, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/lib/python3.7/site-packages/flask/_compat.py", line 35, in reraise
raise value
File "/usr/lib/python3.7/site-packages/flask/app.py", line 1813, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/lib/python3.7/site-packages/flask/app.py", line 1799, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/kleratoni/bot/bot/Main.py", line 90, in webhook
bot.process_new_updates([update])
File "/home/*accountName*/.local/lib/python3.7/site-packages/telebot/__init__.py", line 326, in process_new_updates
self.process_new_callback_query(new_callback_querys)
File "/home/*accountName*/.local/lib/python3.7/site-packages/telebot/__init__.py", line 354, in process_new_callback_query
self._notify_command_handlers(self.callback_query_handlers, new_callback_querys)
File "/home/*accountName*/.local/lib/python3.7/site-packages/telebot/__init__.py", line 1490, in _notify_command_handlers
self._exec_task(message_handler['function'], message)
File "/home/*accountName*/.local/lib/python3.7/site-packages/telebot/__init__.py", line 464, in _exec_task
task(*args, **kwargs)
File "/home/*accountName*/bot/bot/Main.py", line 535, in callback_inline
MainMenuActions.BetMenu_BackButton(call.message, bot, db)
File "/home/*accountName*/bot/bot/MainMenuActions.py", line 130, in BetMenu_BackButton
bot.send_message(message.chat.id, text=config.StringContent["StartPlayWords"], reply_markup=keyborads.keyboardMainmenu)
File "/home/*accountName*/.local/lib/python3.7/site-packages/telebot/__init__.py", line 598, in send_message
reply_markup, parse_mode, disable_notification))
File "/home/*accountName*/.local/lib/python3.7/site-packages/telebot/apihelper.py", line 140, in send_message
return _make_request(token, method_url, params=payload, method='post')
File "/home/*accountName*/.local/lib/python3.7/site-packages/telebot/apihelper.py", line 54, in _make_request
timeout=(connect_timeout, read_timeout), proxies=proxy)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 512, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3.7/site-packages/requests/sessions.py", line 622, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3.7/site-packages/requests/adapters.py", line 445, in send
timeout=timeout
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.7/http/client.py", line 1321, in getresponse
response.begin()
File "/usr/lib/python3.7/http/client.py", line 296, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.7/http/client.py", line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py", line 300, in recv_into
return self.recv_into(*args, **kwargs)
File "/usr/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py", line 285, in recv_into
return self.connection.recv_into(*args, **kwargs)
File "/usr/lib/python3.7/site-packages/OpenSSL/SSL.py", line 1814, in recv_into
self._raise_ssl_error(self._ssl, result)
File "/usr/lib/python3.7/site-packages/OpenSSL/SSL.py", line 1639, in _raise_ssl_error
_raise_current_error()
File "/usr/lib/python3.7/site-packages/OpenSSL/_util.py", line 54, in exception_from_error_queue
raise exception_type(errors)
OpenSSL.SSL.Error: [('SSL routines', 'ssl3_get_record', 'decryption failed or bad record mac')]
From an issue on the pyTelegramBotAPI github repo, it looks like it's an incompatibility between the version of requests you're using and the version of OpenSSL you're using. Try using requests 2.10.0 instead.
I had a same error and I fixed like this
token = 'bot_token'
bot = telebot.TeleBot(token, threaded=False)
I tried peewee with flask for two days, but I failed till now. The code is as the follows:
import click
from flask import Flask
from flask.cli import FlaskGroup
from playhouse.flask_utils import FlaskDB
from models import *
from config import config
flask_db = FlaskDB()
def create_app(config_name):
application = Flask(__name__)
application.config.from_object(config[config_name])
flask_db.init_app(application)
flask_db.connect_db()
flask_db.database.create_tables([User])
flask_db.database.close()
#application.route('/')
def index():
return "hello world!"
return application
def create_cli_app(info):
return create_app("develop")
#click.group(cls=FlaskGroup, create_app=create_cli_app)
def cli():
pass
#cli.command()
def initdb():
flask_db.connect_db()
flask_db.database.create_tables([User])
flask_db.database.close()
if __name__ == "__main__":
cli()
When I run it with the CLI: python manage.py run, I got the following errors:
(venv) ➜ /Users/yw/Documents/web git:(master) ✗ p manage.py run
Traceback (most recent call last): File "manage.py", line 46, in
cli() File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/click/core.py",
line 716, in call
return self.main(*args, **kwargs) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/flask/cli.py",
line 345, in main
return AppGroup.main(self, *args, **kwargs) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/click/core.py",
line 696, in main
rv = self.invoke(ctx) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/click/core.py",
line 1060, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/click/core.py",
line 889, in invoke
return ctx.invoke(self.callback, **ctx.params) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/click/core.py",
line 534, in invoke
return callback(*args, **kwargs) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/click/decorators.py",
line 64, in new_func
return ctx.invoke(f, obj, *args[1:], **kwargs) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/click/core.py",
line 534, in invoke
return callback(*args, **kwargs) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/flask/cli.py",
line 388, in run_command
app = DispatchingApp(info.load_app, use_eager_loading=eager_loading) File
"/Users/yw/Documents/web/venv/lib/python3.4/site-packages/flask/cli.py",
line 124, in init
self._load_unlocked() File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/flask/cli.py",
line 148, in _load_unlocked
self._app = rv = self.loader() File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/flask/cli.py",
line 201, in load_app
rv = self.create_app(self) File "manage.py", line 30, in create_cli_app
return create_app("develop") File "manage.py", line 19, in create_app
flask_db.database.create_tables([User]) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/peewee.py",
line 3765, in create_tables
create_model_tables(models, fail_silently=safe) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/peewee.py",
line 5175, in create_model_tables
m.create_table(**create_table_kwargs) File "/Users/yw/Documents/web/venv/lib/python3.4/site-packages/peewee.py",
line 4845, in create_table
if db.sequences and pk is not False and pk.sequence: AttributeError: 'FlaskDB' object has no attribute 'sequences'
Indeed, I just want to initiate the DB by using flask.cli tool. As you see, if I use the command “python manage.py initdb”, I can only get the same error output as above.
So what is the meaning of "'FlaskDB' object has no attribute ‘sequences'"? What should I do?
Thanks for your help!
It seems that your User model doesn't have correct database.
With FlaskDB, your User model should inherit FlaskDB().Model
instead of defining class Meta: database = database.
database = FlaskDB()
class User(database.Model):
pass
I am starting a brand new project that involves Flask + SQLAlchemy with pymysql. Currently, creation of tables based on model fails with the following:
The lines in my code (main.py) leading up to the error are:
app = Flask(__name__)
app.config['DEBUG'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://myapp:myapp#localhost/myapp'
db = SQLAlchemy(app)
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
guid = db.Column(db.String(255), unique=True)
def __init__(self):
self.guid = str(uuid.uuid1())
db.create_all()
# Note: We don't need to call run() since our application is embedded within
# the App Engine WSGI application server.
#app.route('/')
def hello():
"""Return a friendly HTTP greeting."""
return 'Hello World!'
#app.errorhandler(404)
def page_not_found(e):
"""Return a custom 404 error."""
return 'Sorry, nothing at this URL.', 404
Traceback (most recent call last):
File "/Users/myuserid/Documents/Development/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
File "/Users/myuserid/Documents/Development/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
handler, path, err = LoadObject(self._handler)
File "/Users/myuserid/Documents/Development/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
obj = __import__(path[0])
File "/Users/myuserid/Documents/Development/myapp-server/main.py", line 9, in <module>
db.create_all()
File "/Users/myuserid/Documents/Development/myapp-server/lib/flask_sqlalchemy/__init__.py", line 895, in create_all
self._execute_for_all_tables(app, bind, 'create_all')
File "/Users/myuserid/Documents/Development/myapp-server/lib/flask_sqlalchemy/__init__.py", line 887, in _execute_for_all_tables
op(bind=self.get_engine(app, bind), **extra)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/sql/schema.py", line 3614, in create_all
tables=tables)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/engine/base.py", line 1850, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/engine/base.py", line 1843, in _optional_conn_ctx_manager
with self.contextual_connect() as conn:
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/engine/base.py", line 2034, in contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/engine/base.py", line 2069, in _wrap_pool_connect
return fn()
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 376, in connect
return _ConnectionFairy._checkout(self)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 708, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 480, in checkout
rec = pool._do_get()
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 1049, in _do_get
self._dec_overflow()
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 1046, in _do_get
return self._create_connection()
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 323, in _create_connection
return _ConnectionRecord(self)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 449, in __init__
self.connection = self.__connect()
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/pool.py", line 602, in __connect
connection = self.__pool._invoke_creator(self)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/engine/strategies.py", line 97, in connect
return dialect.connect(*cargs, **cparams)
File "/Users/myuserid/Documents/Development/myapp-server/lib/sqlalchemy/engine/default.py", line 377, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/Users/myuserid/Documents/Development/myapp-server/lib/pymysql/__init__.py", line 88, in Connect
return Connection(*args, **kwargs)
File "/Users/myuserid/Documents/Development/myapp-server/lib/pymysql/connections.py", line 644, in __init__
self._connect()
File "/Users/myuserid/Documents/Development/myapp-server/lib/pymysql/connections.py", line 837, in _connect
self._get_server_information()
File "/Users/myuserid/Documents/Development/myapp-server/lib/pymysql/connections.py", line 1048, in _get_server_information
packet = self._read_packet()
File "/Users/myuserid/Documents/Development/myapp-server/lib/pymysql/connections.py", line 882, in _read_packet
packet_header = self._read_bytes(4)
File "/Users/myuserid/Documents/Development/myapp-server/lib/pymysql/connections.py", line 899, in _read_bytes
data = self._rfile.read(num_bytes)
File "/Users/myuserid/Documents/Development/myapp-server/lib/pymysql/_socketio.py", line 64, in readinto
n = e.args[0]
IndexError: tuple index out of range
INFO 2015-05-05 23:30:40,803 module.py:788] default: "GET / HTTP/1.1" 500 -
I have Flask application and implemented RPC in it.
engine.py
import pickle
import pika
from databasyfacade.rpc import api
__author__ = 'Marboni'
def on_request(ch, method, props, body):
request = pickle.loads(body)
func = getattr(api, request['func'])
try:
result = func(*request['args'])
except Exception, e:
response = {
'status': 'ERROR',
'error': e
}
else:
response = {
'status': 'OK',
'result': result
}
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(
correlation_id=props.correlation_id
),
body=pickle.dumps(response)
)
ch.basic_ack(delivery_tag=method.delivery_tag)
def init(host):
connection = pika.BlockingConnection(pika.ConnectionParameters(host))
channel = connection.channel()
channel.queue_declare(queue='facade_rpc')
channel.basic_qos(prefetch_count=1)
channel.basic_consume(on_request, queue='facade_rpc')
api.py
from databasyfacade.services import profiles_service
__author__ = 'Marboni'
def profile(user_id):
return profiles_service.profile(user_id)
When Flask application initializes, it runs method init(host).
Now I need to test how my application responses to RPC calls. So I wrote following RPC client:
client.py
import pickle
import uuid
import pika
__author__ = 'Marboni'
class RpcClient(object):
def __init__(self, host):
self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=host))
self.channel = self.connection.channel()
result = self.channel.queue_declare(exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(self.on_response, no_ack=True, queue=self.callback_queue)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = body
def call(self, func, *args):
request = {
'func': func,
'args': args
}
self.response = None
self.corr_id = str(uuid.uuid4())
self.channel.basic_publish(exchange='',
routing_key='facade_rpc',
properties=pika.BasicProperties(
reply_to = self.callback_queue,
correlation_id = self.corr_id,
),
body=pickle.dumps(request))
while self.response is None:
self.connection.process_data_events()
response = pickle.loads(self.response)
if response['status'] == 'ERROR':
e = response['error']
raise e
else:
return response['result']
Then I wrote test based on Flask-Testing framework. It initializes Flask application between each test method, so we can interact with it.
tests.py
from databasyfacade.rpc import RpcClient
from databasyfacade.testing import DatabasyTest, fixtures
from databasyfacade.testing.testdata import UserData, ProfileData
__author__ = 'Marboni'
class RpcTest(DatabasyTest):
#fixtures(UserData, ProfileData)
def test_profile(self, data):
rpc = RpcClient(self.app.config['RABBITMQ_HOST'])
profile = rpc.call('profile', ProfileData.hero.user_id)
self.assertIsNotNone(profile)
self.assertEqual(ProfileData.hero.email, profile.email)
This test hangs when makes call. It iterates infinitely here:
From client.py
while self.response is None:
self.connection.process_data_events()
It means that on_response() method on client never called.
If I interrupt my tests with CTRL-C, I will see following stacktrace:
Traceback (most recent call last):
File "../env/bin/nosetests", line 8, in <module>
load_entry_point('nose==1.3.0', 'console_scripts', 'nosetests')()
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/core.py", line 118, in __init__
**extra_args)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/main.py", line 95, in __init__
self.runTests()
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/core.py", line 197, in runTests
result = self.testRunner.run(self.test)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/core.py", line 61, in run
test(result)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
return self.run(*arg, **kw)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 223, in run
test(orig)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/suite.py", line 65, in __call__
return self.run(*args, **kwds)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 74, in run
test(result)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
return self.run(*arg, **kw)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 223, in run
test(orig)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
return self.run(*arg, **kw)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 223, in run
test(orig)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
return self.run(*arg, **kw)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 223, in run
test(orig)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
return self.run(*arg, **kw)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 223, in run
test(orig)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 176, in __call__
return self.run(*arg, **kw)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/suite.py", line 223, in run
test(orig)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/case.py", line 45, in __call__
return self.run(*arg, **kwarg)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/case.py", line 133, in run
self.runTest(result)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/nose/case.py", line 151, in runTest
test(result)
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/flask_testing.py", line 72, in __call__
self._pre_setup()
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/flask_testing.py", line 80, in _pre_setup
self.app = self.create_app()
File "/Users/Marboni/Projects/Databasy/databasy-facade/databasyfacade/databasyfacade/testing/__init__.py", line 12, in create_app
return app.create_app()
File "/Users/Marboni/Projects/Databasy/databasy-facade/databasyfacade/databasyfacade/app.py", line 75, in create_app
init_rpc(app)
File "/Users/Marboni/Projects/Databasy/databasy-facade/databasyfacade/databasyfacade/app.py", line 63, in init_rpc
rpc.init(app.config['RABBITMQ_HOST'])
File "/Users/Marboni/Projects/Databasy/databasy-facade/databasyfacade/databasyfacade/rpc/engine.py", line 39, in init
channel.basic_consume(on_request, queue='facade_rpc')
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/pika/channel.py", line 220, in basic_consume
{'consumer_tag': consumer_tag})])
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 1104, in _rpc
self._wait_on_response(method_frame))
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 1124, in _send_method
self.connection.process_data_events()
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 215, in process_data_events
if self._handle_read():
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 327, in _handle_read
if self._read_poller.ready():
File "/Users/Marboni/Projects/Databasy/databasy-facade/env/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 66, in ready
self.poll_timeout)
KeyboardInterrupt
I tried to run application and access it from separate script:
#!/usr/bin/env python
from databasyfacade.rpc.client import RpcClient
rpc = RpcClient('localhost')
profile = rpc.call('profile', 4L)
print profile.email
As you can see, code is the same as in test, but in this case it works.
What can be the cause of this issue? May be, it's because Flask-Testing runs both application and client in one process? How to check it / write correct test?
I found the cause, it's not related with MQ. Remove method accessed database using SQLAlchemy with scoped_session. Problem disappeared after I finalized session properly:
Session.remove()