While trying to run the Dash app I'm working in debug mode, the following unexpected error is thrown:
* Serving Flask app 'app'
* Debug mode: on
Traceback (most recent call last):
File "/..../anaconda3/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/..../anaconda3/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/...../my-app/app.py", line 1, in <module>
import dash
ModuleNotFoundError: No module named 'dash'
Application works fine in run-mode. Debug-mode used to work as well. Can't figure out what might have changed. My app is a multi-page app with the following basic set-up in app.py (main page)
import dash
from dash import html
import dash_bootstrap_components as dbc
app = dash.Dash(__name__, use_pages=True, suppress_callback_exceptions=True)
...
if __name__ == "__main__":
app.run(debug=True)
Hope my question is clear enough. Many thanks in advance.
Related
I'm trying to import in views.py of my Django app, a python file "load_model.py" which contains my custom pyspark API but I got an error And I can't figure out how to solve it.
I import the file "load-model.py" with a simple:
import load_model as lm
My load_model.py contains the following code (this is just part of the code):
import findspark
# findspark.init('/home/student/spark-2.1.1-bin-hadoop2.7')
findspark.init('/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7')
from pyspark.sql import SparkSession
from pyspark.ml.regression import RandomForestRegressionModel
from pyspark.ml.linalg import Vectors
from pyspark.ml.feature import VectorAssembler
from pyspark.sql import Row
from collections import OrderedDict
spark = SparkSession.builder.appName('RForest_Regression').getOrCreate()
sc = spark.sparkContext
model = RandomForestRegressionModel.load('model/')
def predict(df):
predictions = model.transform(df)
return int(predictions.select('prediction').collect()[0].prediction)
# etc... ... ...
when I lunch python manage.py run server on my command line, I get this error log:
19/07/20 07:22:06 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Traceback (most recent call last):
File "manage.py", line 21, in <module>
main()
File "manage.py", line 17, in main
execute_from_command_line(sys.argv)
File "/anaconda3/lib/python3.7/site-packages/django/core/management/__init__.py", line 381, in execute_from_command_line
utility.execute()
File "/anaconda3/lib/python3.7/site-packages/django/core/management/__init__.py", line 375, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/anaconda3/lib/python3.7/site-packages/django/core/management/base.py", line 323, in run_from_argv
self.execute(*args, **cmd_options)
File "/anaconda3/lib/python3.7/site-packages/django/core/management/commands/runserver.py", line 60, in execute
super().execute(*args, **options)
File "/anaconda3/lib/python3.7/site-packages/django/core/management/base.py", line 364, in execute
output = self.handle(*args, **options)
File "/anaconda3/lib/python3.7/site-packages/django/core/management/commands/runserver.py", line 95, in handle
self.run(**options)
File "/anaconda3/lib/python3.7/site-packages/django/core/management/commands/runserver.py", line 102, in run
autoreload.run_with_reloader(self.inner_run, **options)
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 585, in run_with_reloader
start_django(reloader, main_func, *args, **kwargs)
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 570, in start_django
reloader.run(django_main_thread)
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 288, in run
self.run_loop()
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 294, in run_loop
next(ticker)
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 334, in tick
for filepath, mtime in self.snapshot_files():
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 350, in snapshot_files
for file in self.watched_files():
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 249, in watched_files
yield from iter_all_python_module_files()
File "/anaconda3/lib/python3.7/site-packages/django/utils/autoreload.py", line 101, in iter_all_python_module_files
modules_view = sorted(list(sys.modules.items()), key=lambda i: i[0])
RuntimeError: dictionary changed size during iteration
Exception ignored in: <function JavaWrapper.__del__ at 0x11d2de6a8>
Traceback (most recent call last):
File "/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7/python/pyspark/ml/wrapper.py", line 41, in __del__
File "/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 2000, in detach
File "/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1298, in _detach
File "/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 628, in _garbage_collect_object
File "/anaconda3/lib/python3.7/logging/__init__.py", line 1370, in debug
File "/anaconda3/lib/python3.7/logging/__init__.py", line 1626, in isEnabledFor
TypeError: 'NoneType' object is not callable
Exception ignored in: <function GatewayConnection.__init__.<locals>.<lambda> at 0x11da84d90>
Traceback (most recent call last):
File "/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1061, in <lambda>
File "/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 640, in _garbage_collect_connection
File "/Users/fabiomagarelli/spark-2.4.3-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 487, in quiet_shutdown
File "/anaconda3/lib/python3.7/logging/__init__.py", line 1370, in debug
File "/anaconda3/lib/python3.7/logging/__init__.py", line 1626, in isEnabledFor
TypeError: 'NoneType' object is not callable
pySpark is installed on my computer, I was using it on my jupyter notebook for fitting the model so I don't think the problem is that pyspark is not installed. Any suggestions?
So I found some tutorials on how to deploy a pySpark ML model using databricks, tensor flow etc. All too complex solutions for my limited pySpark knowledge and a project deadline in 4 weeks.
However, I found a workaround which consists in "deploying" the ML model on a Flask App then call it from my Django App (my project app). I think this may be very useful for someone facing my same problem. Not the best practice maybe but working! That's why I'm going to explain each step:
1. Create a Flask Application
in the command line (in your virtual env if you have one), type: pip install flask.
make a new folder (i call it 'static') and place in it the model folder which is obtained by saving the pySpark model (it contains other folders: data, metadata...)
create a new folder for your flask app (can be in the parent folder of your django app) and create a file in it named main.py (you can use whatever name but for the code I'm gonna post, this is the name I used).
in main.py, copy paste this:
from flask import Flask, request
import findspark
findspark.init('/home/student/spark-2.1.1-bin-hadoop2.7')
# various pySpark imports here...
app = Flask(__name__)
spark = SparkSession.builder.appName('RForest_Regression').getOrCreate()
sc = spark.sparkContext
# I'm using a RandomForest ML model, change it as appropriate
model = RandomForestRegressionModel.load('static/model/')
# define here all your functions to make a prediction (eventual arguments cleaning...)
#app.route('/predict')
# this is the function called when the page: '127.0.0.1/5000/predict' is requested.
# you can pass arguments in here by calling: '127.0.0.1/5000/predict?data=...'
numbers = request.args.get('data') # numbers = '...'
makePredictions(numbers)
def makePredictions(n):
# your function here
now on the django app, open views.py
add the fucntion to request the predictions from the flask app:
# Send a request to the flask App where the model is hosted
def getPredictions(request):
try:
data_to_predict = request.GET['data']
url = 'http://127.0.0.1:5000/predict?data=%s' % data_to_predict
response = get(url)
return JsonResponse(response.text, safe=False)
except:
print('ERROR getPredictions: no pySpark module or Flask App not running or wrong arguments')
then call the getPredictions function from javascript in your django app (I haven't done it yet so I don't have a snippet but so far is working, I tested it passing custom arguments).
You have to remember to run the flask app and the django app together in order to make it works:
cd into your Flask app folder (where you have the main.py file) then type: export FLASK_APP=main.py and flask run
then cd into your django app (where you have the manage.py file) then type: python manage.py runserver
I hope this will be useful to someone and that my explaination is not too messy. I will appreciate any comments, suggestions and requests. :)
I have a draft of the Django project with the added one application (e.g. my_app). In this app I have places tests.py file with one test:
import unittest
class Test_MyModel(unittest.TestCase):
def test_dummy(self):
self.assertEqual(1,1)
In this case, this dummy test is discoverd by Visual Studio Code and could be executed in it, also it's possible to launch this test from command line:
python manage.py test
When I modify my file with unit test add import some model (placed in file models.py) from my_app:
import unittest
from .models import MyModel # new added line
class Test_MyModel(unittest.TestCase):
def test_dummy(self):
self.assertEqual(1,1)
In this case I'm still be able run test from command line without any problems, but VSC doesn't discover my test and in the Python Test Log in the VSC I get error:
======================================================================
ERROR: tests (unittest.loader._FailedTest)
----------------------------------------------------------------------
ImportError: Failed to import test module: tests
Traceback (most recent call last):
File "/usr/local/Cellar/python/3.7.1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/loader.py", line 434, in _find_test_path
module = self._get_module_from_name(name)
File "/usr/local/Cellar/python/3.7.1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/loader.py", line 375, in _get_module_from_name
__import__(name)
File "/Users/myuser/Projects/backend/my_app/tests.py", line 3, in <module>
from my_app.models import MyModel
File "/Users/myuser/Projects/backend/my_app/models.py", line 4, in <module>
class MyModel(models.Model):
File "/Users/myuser/Projects/virtualenvs/my_app_env/lib/python3.7/site-packages/django/db/models/base.py", line 103, in __new__
app_config = apps.get_containing_app_config(module)
File "/Users/myuser/Projects/virtualenvs/my_app_env/lib/python3.7/site-packages/django/apps/registry.py", line 252, in get_containing_app_config
self.check_apps_ready()
File "/Users/myuser/Projects/virtualenvs/my_app_env/lib/python3.7/site-packages/django/apps/registry.py", line 134, in check_apps_ready
settings.INSTALLED_APPS
File "/Users/myuser/Projects/virtualenvs/my_app_env/lib/python3.7/site-packages/django/conf/__init__.py", line 79, in __getattr__
self._setup(name)
File "/Users/myuser/Projects/virtualenvs/my_app_env/lib/python3.7/site-packages/django/conf/__init__.py", line 64, in _setup
% (desc, ENVIRONMENT_VARIABLE))
django.core.exceptions.ImproperlyConfigured: Requested setting INSTALLED_APPS, but settings are not configured. You must either define the environment variable DJANGO_SETTINGS_MODULE or call settings.configure() before accessing settings.
Any hints or advice how can I resolve this problem and start fully using unit test under VSC for models test?
I've found a solution to my problem. I need to add below lines into my init.py file in my_app folder.
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "my_app_project.settings")
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
The following error is getting while attempting to run a sample Python Flask application.
Traceback (most recent call last):
File "/home/bineeth/Projects/python/xcontent/bin/flask", line 7, in <module>
from flask.cli import main
File "/home/bineeth/Projects/python/xcontent/lib/python3.5/site-packages/flask/__init__.py", line 18, in <module>
from werkzeug.exceptions import abort
File "/home/bineeth/Projects/python/xcontent/lib/python3.5/site-packages/werkzeug/__init__.py", line 152, in <module>
__import__('werkzeug.exceptions')
File "/home/bineeth/Projects/python/xcontent/lib/python3.5/site-packages/werkzeug/exceptions.py", line 66, in <module>
from werkzeug.http import HTTP_STATUS_CODES
File "/home/bineeth/Projects/python/xcontent/lib/python3.5/site-packages/werkzeug/http.py", line 31, in <module>
from werkzeug.datastructures import (# noqa
File "/home/bineeth/Projects/python/xcontent/lib/python3.5/site-packages/werkzeug/datastructures.py", line 24, in <module>
from werkzeug.http import dump_options_header, dump_header, generate_etag, \
ImportError: cannot import name 'dump_options_header'
Following is the source code;
from flask import Flask
app = Flask(__name__)
#app.route('/ping')
def ping():
return {'status':'completed'}
if __name__ == '__main__':
app.run(debug=False)
I am using Python 3 and LiClipse IDE. Also Linux Mint is the Operating System.
Appreciate your valuable help in this regard.
I need to package a (rather simple) django application into a standalone executable on windows. After some failed attempts with py2exe (I really don't understand why everybody recommends it for django, it seems extremely complicated to set up) I tried pyinstaller. It does package the application without complaining, but if I try to run it using myapp.exe runserver localhost:8000, I am getting the following error:
Traceback (most recent call last):
File "<string>", line 5, in <module>
File "c:\python27\lib\site-packages\PyInstaller-2.1-py2.7.egg\PyInstaller\loader\pyi_importers.py", line 270, in load_
module
exec(bytecode, module.__dict__)
File "...\build\ck\out00-PYZ.pyz\django.test", line 5, in <module>
File "c:\python27\lib\site-packages\PyInstaller-2.1-py2.7.egg\PyInstaller\loader\pyi_importers.py", line 270, in load_
module
exec(bytecode, module.__dict__)
File "...\build\ck\out00-PYZ.pyz\django.test.client", line 21, in <module>
File "c:\python27\lib\site-packages\PyInstaller-2.1-py2.7.egg\PyInstaller\loader\pyi_importers.py", line 270, in load_
module
exec(bytecode, module.__dict__)
File "...\build\ck\out00-PYZ.pyz\django.db", line 11, in <module>
File "...\build\ck\out00-PYZ.pyz\django.conf", line 52, in __getattr__
File "...\build\ck\out00-PYZ.pyz\django.conf", line 45, in _setup
django.core.exceptions.ImproperlyConfigured: Requested setting DATABASES, but settings are not configured. You must eith
er define the environment variable DJANGO_SETTINGS_MODULE or call settings.configure() before accessing settings.
My manage.py script looks like this:
#!/usr/bin/env python
import os
import sys
import Cookie
import django.test
import HTMLParser
from django.conf import settings
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "myapp.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
Why does it not recognize the environment variable and how can I fix that?
I solved it by importing django.test after the line
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "myapp.settings")
I'm trying to write test for a django plugin called Dynamic Response (available on GitHub), but I'm having some issues. Using mock I'm trying to mock certain Django methods like render_to_response, (i'm not going to test django after all) but when I try to do so, i get an error message saying it can't find settings.py, which it shouldn't as this is a plugin to be used in other projects, not a project in itself. I've tried the following code:
class ResponseTest (unittest.TestCase):
#patch('dynamicresponse.response.render_to_response')
def test_is_testframework_working(self, render_to_response):
from dynamicresponse.response import SerializeOrRender
SerializeOrRender()
Which, as stated, gives an error message. Stubbing HttpResponse, HttpRedirect does not throw these errors...
gives the following stacktrace
Response (unittest.loader.ModuleImportFailure) ... ERROR
======================================================================
ERROR: response (unittest.loader.ModuleImportFailure)
----------------------------------------------------------------------
ImportError: Failed to import test module: response
Traceback (most recent call last):
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/loader.py", line 252, in _find_tests
module = self._get_module_from_name(name)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/loader.py", line 230, in _get_module_from_name
__import__(name)
File "/Users/skinney/Workspace/django-dynamicresponse/tests/response.py", line 5, in <module>
class ResponseTest (unittest.TestCase):
File "/Users/skinney/Workspace/django-dynamicresponse/tests/response.py", line 7, in ResponseTest
#patch('dynamicresponse.response.render_to_response')
File "/Users/skinney/Workspace/django-dynamicresponse/tests/mock.py", line 720, in patch
target = _importer(target)
File "/Users/skinney/Workspace/django-dynamicresponse/tests/mock.py", line 514, in _importer
thing = _dot_lookup(thing, comp, import_path)
File "/Users/skinney/Workspace/django-dynamicresponse/tests/mock.py", line 503, in _dot_lookup
__import__(import_path)
File "/Users/skinney/Workspace/django-dynamicresponse/dynamicresponse/response.py", line 3, in <module>
from django.shortcuts import render_to_response
File "/Library/Python/2.7/site-packages/django/shortcuts/__init__.py", line 10, in <module>
from django.db.models.manager import Manager
File "/Library/Python/2.7/site-packages/django/db/__init__.py", line 14, in <module>
if not settings.DATABASES:
File "/Library/Python/2.7/site-packages/django/utils/functional.py", line 276, in __getattr__
self._setup()
File "/Library/Python/2.7/site-packages/django/conf/__init__.py", line 40, in _setup
raise ImportError("Settings cannot be imported, because environment variable %s is undefined." % ENVIRONMENT_VARIABLE)
ImportError: Settings cannot be imported, because environment variable DJANGO_SETTINGS_MODULE is undefined.
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (errors=1)
You can't test django apps without any project (real or dummy, whatever) having its own settings file.