My code has access to 3 databases. syncdb must create all of the model tables in two of the databases. I am unsure how to do this. The below does not work.
DATABASES = {
'default': {}, # empty. default is required
'db_1': { # want tables created in this
...
},
'db_2': { # want tables created in this
...
},
'other': { # do NOT want tables created in this
...
},
}
router: (a separate router handles auth tables)
import random
class OtherRouter(object):
def db_for_read(self, model, **hints):
return random.choice(['db_1', 'db_2'])
def db_for_write(self, model, **hints):
return "db_1"
def allow_relation(self, obj1, obj2, **hints):
return True
def allow_syncdb(self, db, model):
db_list = ('db_1', 'db_2')
if db in db_list:
return True
return None
Got it. Syncdb has to be run with the db parameter. Also, maybe (not sure it matters) allow_syncdb should return False, not None, in other cases.
syncdb --database='db_1'
syncdb --database='db_1'
Related
I am trying to write some unittests for my flask application and for some reason my database is always empty.
import os
import unittest
import json
from flask_sqlalchemy import SQLAlchemy
from flaskr import create_app
from models import setup_db, Question, Category, db
class TriviaTestCase(unittest.TestCase):
"""This class represents the trivia test case"""
def setUp(self):
"""Define test variables and initialize app."""
self.app = create_app()
self.client = self.app.test_client
self.database_name = "trivia_test"
self.database_path = "postgres://{}/{}".format('localhost:5432', self.database_name)
setup_db(self.app, self.database_path)
self.question = {
'question': 'is Yaser amazing?',
'answer': 'of course, are you crazy?',
'difficulty': 1 ,
'category': 'all'
}
# binds the app to the current context
with self.app.app_context():
self.db = SQLAlchemy()
self.db.init_app(self.app)
# create all tables
self.db.create_all()
def tearDown(self):
"""Executed after reach test"""
pass
def test_get_questions(self):
res = self.client().get('/questions')
data = json.loads(res.data)
print(data)
self.assertEqual(res.status_code, 200)
self.assertEqual(data['success'],True)
self.assertTrue(data['totalQuestions'])
self.assertTrue(data['all_categories'])
I thought that self.question{} would add a row in my database but it doesn't. I am not sure if the syntax is right or if it should be something else. I am following a class example and I am stumped.
It looks like you're adding data to your test class object but not adding to your database. To demonstrate:
# The following adds a dictionary to `self.question`
self.question = {
'question': 'is Yaser amazing?',
'answer': 'of course, are you crazy?',
'difficulty': 1 ,
'category': 'all'
}
# The following assertion should be true
assert isinstance(self.question, dict)
To add to your database you would need to do the following:
def add_q(self, commit=True):
"""
Unpack the stored dictionary into your db model,
then add to your db.
If you'd like to query this object from the db,
you'll need to commit the session.
--> Toggle this feature with the param `commit`
"""
self.db.session.add(Question(**self.question))
# You can also commit outside of the function execution
# Helpful for batch processing
if commit:
self.db.session.commit()
After doing the above, you should be able to query your database for the newly added question. If you're running this test often, you'll likely want to remove the newly added q. Here's a helpful function for that:
def remove_q(self, commit=True):
"""
Query your db for the previously added q. Then
remove from db.
To commit this change, set commit=True (default)
"""
my_q = self.db.session.query(Question).filter_by(**self.question).first()
# now delete
self.db.session.delete(my_q)
# You can also commit outside of the function execution
# Helpful for batch processing
if commit:
self.db.session.commit()
We have a multiple database Django project that uses database routers.
For some reason when we run a migration the reference to the migration end up in the django_migrations table but no actual migrations are actually run - that is - there is no change in target database.
Following is the database router for the elegant database.
class ElegantRouter:
"""
A router to control all database operations on models in the
elegant application.
"""
def db_for_read(self, model, **hints):
"""
Attempts to read elegant models go to elegant.
"""
if model._meta.app_label == 'elegant':
return 'elegant'
return None
def db_for_write(self, model, **hints):
"""
Attempts to write elegant models go to elegant.
"""
if model._meta.app_label == 'elegant':
return 'elegant'
return None
def allow_relation(self, obj1, obj2, **hints):
"""
Allow relations if a model in the elegant app is involved.
"""
if obj1._meta.app_label == 'elegant' or \
obj2._meta.app_label == 'elegant':
return True
return None
def allow_migrate(self, db, app_label, model_name=None, **hints):
"""
Make sure the elegant app only appears in the 'elegant'
database.
"""
print('allow_migrate',app_label,db)
if app_label == 'elegant':
return db == 'elegant'
return None
Following is the DATABASES settings for the project.
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
},
'depot_maestro': {
'ENGINE': 'django_filemaker_pyodbc',
'HOST': os.getenv('FILEMAKER_HOST'),
'PORT': os.getenv('FILEMAKER_PORT'),
'USER': os.getenv('FILEMAKER_USER'),
'PASSWORD': os.getenv('FILEMAKER_PASSWORD'),
'NAME': os.getenv('FILEMAKER_FILENAME'),
'OPTIONS' : {
'driver' : os.getenv('FILEMAKER_DRIVER'),
'driver_supports_utf8' : True,
'autocommit' : True ,
},
'TEST': {
'NAME': os.getenv('FILEMAKER_FILENAME'),
'SERIALIZE': False,
}
},
'postgres': {
'NAME': os.getenv('POSTGRES_DATABASE'),
'ENGINE': 'django.db.backends.postgresql',
'USER': os.getenv('POSTGRES_USER'),
'PASSWORD': os.getenv('POSTGRES_PASSWORD')
},
'elegant': {
'NAME': os.getenv('ELEGANT_DATABASE'),
'ENGINE': 'django.db.backends.postgresql',
'USER': os.getenv('ELEGANT_USER'),
'PASSWORD': os.getenv('ELEGANT_PASSWORD')
},
}
Update
Using sqlmigrate for the project with database routers returns no sql.
./manage.py sqlmigrate elegant 0057 --database elegant
BEGIN;
--
-- Remove field internal_id from organisationelegant
--
--
-- Add field uuid to organisationelegant
--
--
-- Alter field id on organisationelegant
--
COMMIT;
where as the project without database routers returns SQL
./manage.py sqlmigrate elegant 0057
BEGIN;
--
-- Remove field internal_id from organisationelegant
--
ALTER TABLE "organisation" DROP COLUMN "id" CASCADE;
...
--
-- Alter field id on organisationelegant
--
DROP INDEX IF EXISTS "organisation_uuid_19796862_like";
COMMIT;
How can you make a migration with database routers take effect when allow migrate already returns true and the migration files are being produced ?
I ended up creating a new project with the same app referencing the same database and migrated from that project.
Update
I added symbolic links to the main project.
In my Django project I have a couple of applications, one of them is email_lists and this application does a lot of data handling reading data from the Model Customers. In my production environment I have two databases: default and read-replica. I would like all queries in a particular module to be made against the replica-set database.
I can do that if I explicitly tell the query to do so:
def get_customers(self):
if settings.ENV == 'production':
customers = Customer.objects.using('read-replica').filter()
else:
customers = Customer.objects.filter()
but this module has more than 100 queries to the Customer and other models. I also have queries to relations like:
def get_value(self, customer):
target_sessions = customer.sessions.filter(status='open')
carts = Cart.objects.filter(session__in=target_sessions)
the idea is that I want to avoid writing:
if settings.ENV == 'production':
instance = Model.objects.using('read-replica').filter()
else:
instance = Model.objects.filter()
for every query. There are other places in my project that do need to read from default database so it can't be a global setting. I just need this module or file to read using the replica.
Is this possible in Django, are there any shortcuts ?
Thanks
You can read on django database routers for this, some good examples can be found online as well and they should be straightforward.
--
Another solution would be to modify the Model manager.
from django.db import models
class ReplicaRoutingManager(models.Manager):
def get_queryset(self):
queryset = super().get_queryset(self)
if settings.ENV == 'production':
return queryset.using('read-replica')
return queryset
class Customer(models.Model):
...
objects = models.Manager()
replica_objects = ReplicaRoutingManager()
with this, you can just use the normal Customer.objects.filter and the manager should do the routing.
I still suggest going with the database router solution, and creating a custom logic in the class. But if the manager works for you, its fine.
If you want All the queries in the email_lists app to query read-replica, then a router is the way to go. If you need to query different databases within the same app, then #ibaguio's solution is the way to go. Here's a basic router example similar to what I'm using:
project/database_routers.py
MAP = {'some_app': 'default',
'some_other_app': 'default',
'email_lists': 'read-replica',}
class DatabaseRouter:
def db_for_read(self, model, **hints):
return MAP.get(model._meta.app_label, None)
def db_for_write(self, model, **hints):
return MAP.get(model._meta.app_label, None)
def allow_relation(self, object_1, object_2, **hints):
database_object_1 = MAP.get(object_1._meta.app_label)
database_object_2 = MAP.get(object_2._meta.app_label)
return database_object_1 == database_object_2
def allow_migrate(self, db, app_label, model=None, **hints):
return MAP.get(app_label, None)
In settings.py:
DATABASE_ROUTERS = ['project.database_router.DatabaseRouter',]
It looks like you only want it in production, so I would think you could add it conditionally:
if ENV == 'production':
DATABASE_ROUTERS = ['project.database_router.DatabaseRouter',]
Running a database migration with RunPython on a second database fails
python3 manage.py migrate --database=app
The problem is that the apps.get_model method takes the default database which has already the newest migrations.
Does not work:
def copy_cpr_cents_to_euros(apps, schema_editor):
User = apps.get_model('accounting', 'User')
User.objects.filter(...);
Works:
def copy_cpr_cents_to_euros(apps, schema_editor):
User = apps.get_model('accounting', 'User')
User.objects.using('app').filter(...);
Is there a way to use the given database in the migration, so in this case "app" without making expliclitly declaring it, since it should work for both databases?
So something like:
User.objects.using(database_name).filter(...)
schema_editor.connection.alias
contains the string of the current database with which the migration was started.
So each RunPython-migration must use this alias to manually select the right db.
Example:
def copy_cpr_cents_to_euros(apps, schema_editor):
User = apps.get_model('accounting', 'User')
db = schema_editor.connection.alias
User.objects.using('app').using(db).filter(...)
Decorators which can be used for RunPython function to specify against which DB it should be executed [Tested on Django 1.8]
def only_default_db_migration(func):
return only_databases_migration('default')(func)
def only_databases_migration(*db_aliases):
"""Restrict running Data Migrations on wanted databases only"""
def decorate(func):
def run_python_func(apps, schema_editor):
db_alias = schema_editor.connection.alias
if db_alias in db_aliases:
return func(apps, schema_editor)
else:
print(f'Skip RunPython {func.__name__!r} for db with alias {db_alias!r}')
return run_python_func
return decorate
Usage of only_default_db_migration
#only_default_db_migration
def migrate_spam_check_processed_at(apps, schema_editor):
apps.get_model("queues","Queue").objects.all().update(check=F('created'))
class Migration(migrations.Migration):
operations = [
migrations.RunPython(migrate_spam_check_processed_at),
]
I have successfully got my application running over several databases using the routing scheme based on models. I.e. model A lives on DB A and model B lives on DB B. I now need to shard my data. I am looking at the docs and having trouble working out how to do it as the same model needs to exist on multiple database servers. I want to have a flag to say DB for NEW members is now database X and that members X-Y live on database N etc.
How do I do that? Is it using **hints, this seems inadequately documented to me.
The hints parameter is designed to help your database router decide where it should read or write its data. It may evolve with future versions of python, but for now there's just one kind of hint that may be given by the Django framework, and that's the instance it's working on.
I wrote this very simple database router to see what Django does:
# routers.py
import logging
logger = logging.getLogger("my_project")
class DebugRouter(object):
"""A debugging router"""
def db_for_read(self, model, **hints):
logger.debug("db_for_read %s" % repr((model, hints)))
return None
def db_for_write(self, model, **hints):
logger.debug("db_for_write %s" % repr((model, hints)))
return None
def allow_relation(self, obj1, obj2, **hints):
logger.debug("allow_relation %s" % repr((obj1, obj2, hints)))
return None
def allow_syncdb(self, db, model):
logger.debug("allow_syncdb %s" % repr((db, model)))
return None
You declare this in settings.py:
DATABASE_ROUTERS = ["my_project.routers.DebugRouter"]
Make sure logging is properly configured to output debug output (for example to stderr):
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
[...some other handlers...]
'stderr': {
'level': 'DEBUG',
'class': 'logging.StreamHandler'
}
},
'loggers': {
[...some other loggers...]
'my_project': {
'handlers': ['stderr'],
'level': 'DEBUG',
'propagate': True,
},
}
}
Then you can open a Django shell and test a few requests to see what data your router is being given:
$ ./manage.py shell
[...]
>>> from my_project.my_app.models import User
>>> User.objects.get(pk = 1234)
db_for_read (<class 'my_project.my_app.models.User'>, {})
<User: User object>
>>> user = User.objects.create(name = "Arthur", title = "King")
db_for_write (<class 'my_project.my_app.models.User'>, {})
>>> user.name = "Kong"
>>> user.save()
db_for_write (<class 'my_project.my_app.models.User'>, {'instance':
<User: User object>})
>>>
As you can see, the hints is always empty when no instance is available (in memory) yet. So you cannot use routers if you need query parameters (the object's id for example) in order to determine which database to query. It might be possible in the future if Django provides the query or queryset objects in the hints dict.
So to answer your question, I would say that for now you must create a custom Manager, as suggested by Aaron Merriam. But overriding just the create method is not enough, since you also need to be able to fetch an object in the appropriate database. Something like this might work (not tested yet):
class CustomManager(models.Manager)
def self.find_database_alias(self, pk):
return #... implement the logic to determine the shard from the pk
def self.new_object_database_alias(self):
return #... database alias for a new object
def get(self, *args, **kargs):
pk = kargs.get("pk")
if pk is None:
raise Exception("Sharded table: you must provide the primary key")
db_alias = self.find_database_alias(pk)
qs = self.get_query_set().using(db_alias)
return qs.get(*args, **kargs)
def create(self, *args, **kwargs):
db_alias = self.new_object_database_alias()
qs = super(CustomManager, self).using(db_alias)
return qs.create(*args, **kwargs)
class ModelA(models.Model):
objects = CustomManager()
Cheers
using should allow you to designate which database you want to use.
subclassing the create method might accomplish what you're looking to do.
class CustomManager(models.Manager)
def get_query_set(self):
return super(CustomManager, self).get_query_set()
def create(self, *args, **kwargs):
return super(CustomManager, self).using('OTHER_DB').create(*args, **kwargs)
class ModelA(models.Model):
objects = CustomManager()
I have not tested this so I don't know if you can tack a 'create' onto the end of a 'using'