Remove old permissions in django - django

In my Django site there are some permissions entries linked to applications that I've removed. For example I have permissions entries linked to "Dashboard" and "Jet" applications. How can you remove them?

Permissions have foreign keys to content types under the hood, so removing the content types for the models that no longer exist will also remove the permissions for those models.
Fortunately, Django also provides a manage.py command to remove old content types: remove_stale_contenttypes. Running that command will list the content types that no longer exist and the related objects (including permissions) that will be deleted, allowing you to review the changes and approve them.
$ manage.py remove_stale_contenttypes
Some content types in your database are stale and can be deleted.
Any objects that depend on these content types will also be deleted.
The content types and dependent objects that would be deleted are:
- Content type for stale_app.removed_model
- 4 auth.Permission object(s)
This list doesn't include any cascade deletions to data outside of Django's
models (uncommon).
Are you sure you want to delete these content types?
If you're unsure, answer 'no'.
Type 'yes' to continue, or 'no' to cancel:

To start, make an empty migration file:
python manage.py makemigrations --empty yourappname
Change the migration (this is an example, adjust to your needs):
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
def add_permissions(apps, schema_editor):
pass
def remove_permissions(apps, schema_editor):
"""Reverse the above additions of permissions."""
ContentType = apps.get_model('contenttypes.ContentType')
Permission = apps.get_model('auth.Permission')
content_type = ContentType.objects.get(
model='somemodel',
app_label='yourappname',
)
# This cascades to Group
Permission.objects.filter(
content_type=content_type,
codename__in=('add_somemodel', 'change_somemodel', 'delete_somemodel'),
).delete()
class Migration(migrations.Migration):
dependencies = [
('yourappname', '0001_initial'),
]
operations = [
migrations.RunPython(remove_permissions, add_permissions),
]

I did it this way:
import re
for perm in Permission.objects.all():
if re.match( r".+modelname.+permissionname.+",str(perm)):
print(perm)
perm.delete()

If you have custom or model based (default) permissions you wish to remove you could write a command like this to accomplish this task:
from django.conf import settings
from django.contrib.auth.models import Permission
from django.core.management.base import BaseCommand
import django.apps
class Command(BaseCommand):
help = 'Remove custom permissions that are no longer in models'
def handle(self, *args, **options):
# get the db name needed for removal...
database_name = input('Database Name: ')
default_perm_names = list()
# are real perms in db, may not be accurate
db_custom_perm_names = list()
# will be used to ensure they are correct.
meta_custom_perm_names = list()
default_and_custom_perms = list()
for model in django.apps.apps.get_models():
# add to models found to fix perms from removed models
app_label = model._meta.app_label
lower_model_name = model._meta.model_name
all_model_permissions = Permission.objects.using(database_name).filter(content_type__app_label=app_label, content_type__model=lower_model_name)
default_and_custom_perms.extend([x for x in all_model_permissions])
# get the custom meta permissions, these should be in the meta of the class
# will be a list or tuple or list, [0=codename, 1=name]
meta_permissions = model._meta.permissions
if meta_permissions:
for perm in all_model_permissions:
# will be the model name from the content type, this is how django makes default perms
# we are trying to remove them so now we can figure out which ones are default by provided name
model_name_lower = perm.content_type.name
# default_perms = ['add', 'change', 'view', 'delete', 'undelete']
# append them to the list of default names
default_perm_names.append(f'Can add {model_name_lower}')
default_perm_names.append(f'Can change {model_name_lower}')
default_perm_names.append(f'Can view {model_name_lower}')
default_perm_names.append(f'Can delete {model_name_lower}')
default_perm_names.append(f'Can undelete {model_name_lower}')
# will mean this is a custom perm...so add it
if not perm.name in default_perm_names:
db_custom_perm_names.append(perm.name)
# the perms to ensure are correct...
for model_perm in meta_permissions:
# get the meta perm, will be a list or tuple or list, [0=codename, 1=name]
custom_perm = Permission.objects.using(database_name).get(codename=model_perm[0], name=model_perm[1])
meta_custom_perm_names.append(custom_perm.name)
perms_to_remove = [perm for perm in db_custom_perm_names if perm not in meta_custom_perm_names]
if not perms_to_remove:
print('There are no stale custom permissions to remove.')
# print(perms_to_remove)
# now remove the custom permissions that were removed from the model
for actual_permission_to_remove in Permission.objects.using(database_name).filter(name__in=perms_to_remove):
# print(actual_permission_to_remove)
actual_permission_to_remove.delete(using=database_name)
print(actual_permission_to_remove, '...deleted')
for perm in [x for x in Permission.objects.using(database_name)]:
# loop all perms...if it is not in the model perms it does not exist...
if perm.content_type.model not in [x.content_type.model for x in default_and_custom_perms]:
perm.delete(using=database_name)
print(perm, 'regular permission...deleted')
If you also wish to ensure that the default permissions are added from Django you can add this snippet in the command:
from django.apps import apps
from django.contrib.auth.management import create_permissions
from apps.client.models import ClientInformation
# add all permissions the django way
# get the db name needed from settings.py
database_name = 'default' # or whatever DB you are looking for
print(f'adding all permissions if not there to {database_name}')
for app_config in apps.get_app_configs():
# print(app_config)
app_config.models_module = True
create_permissions(app_config, using=database_name)
app_config.models_module = None
Then call via python manage.py fix_permissions if you name your command file fix_permissions.py

I've reworked #ViaTech's code to use Django's contrib.auth.management._get_all_permissions() functions which makes it more straight forward:
from typing import List, Set, Tuple
import django.apps
# noinspection PyProtectedMember
from django.contrib.auth.management import _get_all_permissions
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand
from django.db import DEFAULT_DB_ALIAS
class Command(BaseCommand):
help = "Remove custom permissions that are no longer defined in models"
def add_arguments(self, parser):
parser.add_argument(
"--database",
default=DEFAULT_DB_ALIAS,
help=f'Specifies the database to use. Default is "{DEFAULT_DB_ALIAS}".',
)
parser.add_argument(
"--dry",
action="store_true",
help="Do a dry run not actually deleting any permissions",
)
def handle(self, *args, **options) -> str:
using = options["database"]
# This will hold the permissions that models have defined,
# i.e. default permissions plus additional custom permissions:
# (content_type.pk, codename)
defined_perms: List[Tuple[int, str]] = []
for model in django.apps.apps.get_models():
ctype = ContentType.objects.db_manager(using).get_for_model(
model, for_concrete_model=False
)
# noinspection PyProtectedMember
for (codename, _) in _get_all_permissions(model._meta):
defined_perms.append((ctype.id, codename))
# All permissions in current database (including stale ones)
all_perms = Permission.objects.using(using).all()
stale_perm_pks: Set[int] = set()
for perm in all_perms:
if (perm.content_type.pk, perm.codename) not in defined_perms:
stale_perm_pks.add(perm.pk)
self.stdout.write(f"Delete permission: {perm}")
# Delete all stale permissions
if options["dry"]:
result = f"DRY RUN: {len(stale_perm_pks)} stale permissions NOT deleted"
else:
if stale_perm_pks:
Permission.objects.filter(pk__in=stale_perm_pks).delete()
result = f"{len(stale_perm_pks)} stale permissions deleted"
return result

Related

how to keep data created in the ready method? Django production vs test database

As you know django give you clear database in testing, but I have a ready() method that create some data for me and I need to query these data in my tests.
class YourAppConfig(AppConfig):
default_auto_field = 'django.db.models.AutoField'
name = 'Functions.MyAppsConfig'
def ready(self):
from django.contrib.auth.models import Permission
from django import apps
from django.contrib.contenttypes.models import ContentType
try:
Permission.objects.get_or_create(....)
MyOtherModel.objects.get_or_create(....)
except:
pass
class TestRules(APITestCase):
def test_my_model(self):
....
x = MyOtherModel.objects.filter(....).first()
# x = None # <=========== problem is here ========= I need to see the data that I created in the ready method
....
You can use the fixtures for that, in each Test case you can fixtures to it as stated documentation example is
class Test(TransactionTestCase):
fixtures = ['user-data.json']
def setUp():
…
Django will load the fixtures before under test case

Avoiding circular imports in Django Models (Config class)

I've created a Configuration model in django so that the site admin can change some settings on the fly, however some of the models are reliant on these configurations. I'm using Django 2.0.2 and Python 3.6.4.
I created a config.py file in the same directory as models.py.
Let me paracode (paraphase the code? Real Enum has many more options):
# models.py
from .config import *
class Configuration(models.Model):
starting_money = models.IntegerField(default=1000)
class Person(models.Model):
funds = models.IntegarField(default=getConfig(ConfigData.STARTING_MONEY))
# config.py
from .models import Configuration
class ConfigData(Enum):
STARTING_MONEY = 1
def getConfig(data):
if not isinstance(data, ConfigData):
raise TypeError(f"{data} is not a valid configuration type")
try:
config = Configuration.objects.get_or_create()
except Configuration.MultipleObjectsReturned:
# Cleans database in case multiple configurations exist.
Configuration.objects.exclude(Configuration.objects.first()).delete()
return getConfig(data)
if data is ConfigData.MAXIMUM_STAKE:
return config.max_stake
How can I do this without an import error? I've tried absolute imports
You can postpone loading the models.py by loading it in the getConfig(data) function, as a result we no longer need models.py at the time we load config.py:
# config.py (no import in the head)
class ConfigData(Enum):
STARTING_MONEY = 1
def getConfig(data):
from .models import Configuration
if not isinstance(data, ConfigData):
raise TypeError(f"{data} is not a valid configuration type")
try:
config = Configuration.objects.get_or_create()
except Configuration.MultipleObjectsReturned:
# Cleans database in case multiple configurations exist.
Configuration.objects.exclude(Configuration.objects.first()).delete()
return getConfig(data)
if data is ConfigData.MAXIMUM_STAKE:
return config.max_stake
We thus do not load models.py in the config.py. We only check if it is loaded (and load it if not) when we actually execute the getConfig function, which is later in the process.
Willem Van Onsem's solution is a good one. I have a different approach which I have used for circular model dependencies using django's Applications registry. I post it here as an alternate solution, in part because I'd like feedback from more experienced python coders as to whether or not there are problems with this approach.
In a utility module, define the following method:
from django.apps import apps as django_apps
def model_by_name(app_name, model_name):
return django_apps.get_app_config(app_name).get_model(model_name)
Then in your getConfig, omit the import and replace the line
config = Configuration.objects.get_or_create()
with the following:
config_class = model_by_name(APP_NAME, 'Configuration')
config = config_class.objects.get_or_create()

Sphinx-apidoc strange output for django app/models.py

I get some missed info in generated documentation of django project, for example first_name and last_name, email are missed (although they are defined in a parent abstract class). How to control what gets added into documentation based on sphinx-apidoc scan? My goal is to auto-generate the docs based on documentation, but it seems that sphinx-apidoc is supposed to be used only one time for initial scaffolding
I tried to use :inherited-members: as shown below but it still didn't produce first_name, last_name, email that exist in AbstractUser class
.. automodule:: apps.users.models
:members:
:inherited-members:
:show-inheritance:
I execute the following command
sphinx-apidoc -f -e -d 2 -M -o docs/code apps '*tests*' '*migrations*'
Output
my apps/users/models.py
from django.contrib.auth.models import AbstractUser
from django.contrib.postgres.fields import HStoreField
from imagekit import models as imagekitmodels
from imagekit.processors import ResizeToFill
from libs import utils
# Solution to avoid unique_together for email
AbstractUser._meta.get_field('email')._unique = True
def upload_user_media_to(instance, filename):
"""Upload media files to this folder"""
return '{}/{}/{}'.format(instance.__class__.__name__.lower(), instance.id,
utils.get_random_filename(filename))
__all__ = ['AppUser']
class AppUser(AbstractUser):
"""Custom user model.
Attributes:
avatar (file): user's avatar, cropeed to fill 300x300 px
notifications (dict): settings for notifications to user
"""
avatar = imagekitmodels.ProcessedImageField(
upload_to=upload_user_media_to,
processors=[ResizeToFill(300, 300)],
format='PNG',
options={'quality': 100},
editable=False,
null=True,
blank=False)
notifications = HStoreField(null=True)
# so authentication happens by email instead of username
# and username becomes sort of nick
USERNAME_FIELD = 'email'
# Make sure to exclude email from required fields if authentication
# is done by email
REQUIRED_FIELDS = ['username']
def __str__(self):
return self.username
class Meta:
verbose_name = 'User'
verbose_name_plural = 'Users'
My sphinx conf.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
import django
import sphinx_py3doc_enhanced_theme
sys.path.insert(0, os.path.abspath('../'))
sys.path.insert(0, os.path.abspath('.'))
os.environ.setdefault(
"DJANGO_SETTINGS_MODULE", "config.settings.local")
django.setup()
# Extensions
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.napoleon',
'sphinx.ext.viewcode',
'sphinxcontrib.blockdiag'
]
napoleon_google_docstring = True
napoleon_use_param = True
napoleon_use_ivar = False
napoleon_use_rtype = True
napoleon_include_special_with_doc = False
# RST support
source_suffix = '.rst'
# Name of master doc
master_doc = 'index'
# General information about the project.
project = 'crm'
copyright = '2017, Company'
author = 'Company'
version = '0.1'
release = '0.1'
language = None
exclude_patterns = []
todo_include_todos = False
# Read the docs theme
html_theme = 'sphinx_py3doc_enhanced_theme'
html_theme_path = [sphinx_py3doc_enhanced_theme.get_html_theme_path()]
html_static_path = []
htmlhelp_basename = 'crmdoc'
latex_elements = {}
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'crm', 'crm Documentation',
[author], 1)
]
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'crm', 'crm Documentation',
author, 'crm', 'One line description of project.',
'Miscellaneous'),
]
html_theme_options = {
'githuburl': 'https://github.com/ionelmc/sphinx-py3doc-enhanced-theme/',
'bodyfont': '"Lucida Grande",Arial,sans-serif',
'headfont': '"Lucida Grande",Arial,sans-serif',
'codefont': '"Deja Vu Sans Mono",consolas,monospace,sans-serif',
'linkcolor': '#0072AA',
'visitedlinkcolor': '#6363bb',
'extrastyling': False,
'sidebarwide': True
}
pygments_style = 'friendly'
html_context = {
'css_files': ['_static/custom.css'],
}
Okay turned out that I had to use :undoc-members: but it created a mess.
This is required since django's AbstractUser class is not properly documented and sphinx has to be forced to display fields only with undoc-members defined. But undoc-members cause a mess, so the solution is just to add documentation in docstr of the child class for attributes/methods that have not been documented in parent class, after that my documentation got these fields displayed
class AppUser(AbstractUser):
"""Custom user model.
Attributes:
avatar (file): user's avatar, cropeed to fill 300x300 px
notifications (dict): settings for notifications to user
first_name (str): first name
last_name (str): last name
"""

Programmatically create a django group with permissions

In the Admin console, I can add a group and add a bunch of permissions that relate to my models, e.g.
api | project | Can add project
api | project | Can change project
api | project | Can delete project
How can I do this programmatically. I can't find any information out there on how to do this.
I have:
from django.contrib.auth.models import Group, Permission
from django.contrib.contenttypes.models import ContentType
from api.models import Project
new_group, created = Group.objects.get_or_create(name='new_group')
# Code to add permission to group ???
ct = ContentType.objects.get_for_model(Project)
# Now what - Say I want to add 'Can add project' permission to new_group?
UPDATE: Thanks for the answer you provided. I was able to use that to work out what I needed. In my case, I can do the following:
new_group, created = Group.objects.get_or_create(name='new_group')
proj_add_perm = Permission.objects.get(name='Can add project')
new_group.permissions.add(proj_add_perm)
Use below code
from django.contrib.auth.models import Group, Permission
from django.contrib.contenttypes.models import ContentType
from api.models import Project
new_group, created = Group.objects.get_or_create(name='new_group')
# Code to add permission to group ???
ct = ContentType.objects.get_for_model(Project)
# Now what - Say I want to add 'Can add project' permission to new_group?
permission = Permission.objects.create(codename='can_add_project',
name='Can add project',
content_type=ct)
new_group.permissions.add(permission)
I needed to create a default set of groups and permission (view only) for those groups. I came up with a manage.py command that may be useful to others (create_groups.py). You can add it to your <app>/management/commands dir, and then run via manage.py create_groups:
"""
Create permission groups
Create permissions (read only) to models for a set of groups
"""
import logging
from django.core.management.base import BaseCommand
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
GROUPS = ['developers', 'devops', 'qa', 'operators', 'product']
MODELS = ['video', 'article', 'license', 'list', 'page', 'client']
PERMISSIONS = ['view', ] # For now only view permission by default for all, others include add, delete, change
class Command(BaseCommand):
help = 'Creates read only default permission groups for users'
def handle(self, *args, **options):
for group in GROUPS:
new_group, created = Group.objects.get_or_create(name=group)
for model in MODELS:
for permission in PERMISSIONS:
name = 'Can {} {}'.format(permission, model)
print("Creating {}".format(name))
try:
model_add_perm = Permission.objects.get(name=name)
except Permission.DoesNotExist:
logging.warning("Permission not found with name '{}'.".format(name))
continue
new_group.permissions.add(model_add_perm)
print("Created default group and permissions.")
UPDATE: A bit more sophisticated now:
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType
READ_PERMISSIONS = ['view', ] # For now only view permission by default for all, others include add, delete, change
WRITE_PERMISSIONS = ['add', 'change', 'delete']
EMAIL_USER_DOMAIN = 'your-domain.com'
# Add your groups here, app and model code
GROUP_MODEL = ('auth', 'group')
USER_MODEL = ('auth', 'user')
PERMISSION_MODEL = ('auth', 'permission')
LOG_ENTRY_MODEL = ('admin', 'logentry')
def add_group_permissions(group_names, model_natural_keys, permissions):
"""
Add permissions to the provided groups for the listed models.
Error raised if permission or `ContentType` can't be found.
:param group_names: iterable of group names
:param model_natural_keys: iterable of 2-tuples containing natural keys for ContentType
:param permissions: iterable of str (permission names i.e. add, view)
"""
for group_name in group_names:
group, created = Group.objects.get_or_create(name=group_name)
for model_natural_key in model_natural_keys:
perm_to_add = []
for permission in permissions:
# using the 2nd element of `model_natural_key` which is the
# model name to derive the permission `codename`
permission_codename = f"{permission}_{model_natural_key[1]}"
try:
perm_to_add.append(
Permission.objects.get_by_natural_key(
permission_codename, *model_natural_key
)
)
except Permission.DoesNotExist:
# trying to add a permission that doesn't exist; log and continue
logging.error(
f"permissions.add_group_permissions Permission not found with name {permission_codename!r}."
)
raise
except ContentType.DoesNotExist:
# trying to add a permission that doesn't exist; log and continue
logging.error(
"permissions.add_group_permissions ContentType not found with "
f"natural name {model_natural_key!r}."
)
raise
group.permissions.add(*perm_to_add)
def set_users_group(users, group):
"""
Adds users to specific permission group.
If user or group does not exist, they are created.
Intended for use with special users for api key auth.
:param users: list of str, usernames
:param group: str, group for which users should be added to
:return: list, user objects added to group
"""
users = users or []
user_objs = []
for user_name in users:
try:
user = User.objects.get(username=user_name)
except User.DoesNotExist:
user = User.objects.create_user(username=user_name,
email=f'{user_name}#{EMAIL_USER_DOMAIN}',
password='')
user_objs.append(user)
group, created = Group.objects.get_or_create(name=group)
group.user_set.add(user)
return user_objs
API_READ_GROUP = 'api-read-users'
API_WRITE_GROUP = 'api-write-users'
READ_GROUPS = [API_READ_GROUP, ]
WRITE_GROUPS = [API_WRITE_GROUP, ] # Can be used in same way as read users below
# Adding users to a group
set_users_group(READ_USERS, API_READ_GROUP)
# Setting up the group permissions i.e. read for a group of models
add_group_permissions(READ_GROUPS, [GROUP_MODEL, USER_MODEL, LOG_ENTRY_MODEL], READ_PERMISSIONS)
I also found that using manage.py update_permissions is useful to sort out/clean up stale permissions if models have changed etc.. Its part of django-extensions commands.
Inspired by radtek's answer I created a bit better version (in my opinion).
It allows specifying model as object (instead of string) and specifying all configuration in one dictionary (instead of several lists)
# backend/management/commands/initgroups.py
from django.core.management import BaseCommand
from django.contrib.auth.models import Group, Permission
from backend import models
GROUPS_PERMISSIONS = {
'ConnectionAdmins': {
models.StaticCredentials: ['add', 'change', 'delete', 'view'],
models.NamedCredentials: ['add', 'change', 'delete', 'view'],
models.Folder: ['add', 'change', 'delete', 'view'],
models.AppSettings: ['view'],
},
}
class Command(BaseCommand):
def __init__(self, *args, **kwargs):
super(Command, self).__init__(*args, **kwargs)
help = "Create default groups"
def handle(self, *args, **options):
# Loop groups
for group_name in GROUPS_PERMISSIONS:
# Get or create group
group, created = Group.objects.get_or_create(name=group_name)
# Loop models in group
for model_cls in GROUPS_PERMISSIONS[group_name]:
# Loop permissions in group/model
for perm_index, perm_name in \
enumerate(GROUPS_PERMISSIONS[group_name][model_cls]):
# Generate permission name as Django would generate it
codename = perm_name + "_" + model_cls._meta.model_name
try:
# Find permission object and add to group
perm = Permission.objects.get(codename=codename)
group.permissions.add(perm)
self.stdout.write("Adding "
+ codename
+ " to group "
+ group.__str__())
except Permission.DoesNotExist:
self.stdout.write(codename + " not found")
Taking ideas from the answers of #radtek and #Pavel I created my own version of create_groups.py which I call using python manage.py create_groups. This file is stored on app_name/management/commands/create_groups.py. I created a __init__.py inside each the management and commands folders.
I have created the possibility to control each model permissions separately because I had a group of users called Member that must have different permissions on different models.
I also added the possibility to create users with emails and a default password that would have to be changed afterwards and associate them with a certain group.
from django.core.management import BaseCommand
from django.contrib.auth.models import User, Group , Permission
import logging
GROUPS = {
"Administration": {
#general permissions
"log entry" : ["add","delete","change","view"],
"group" : ["add","delete","change","view"],
"permission" : ["add","delete","change","view"],
"user" : ["add","delete","change","view"],
"content type" : ["add","delete","change","view"],
"session" : ["add","delete","change","view"],
#django app model specific permissions
"project" : ["add","delete","change","view"],
"order" : ["add","delete","change","view"],
"staff time sheet" : ["add","delete","change","view"],
"staff" : ["add","delete","change","view"],
"client" : ["add","delete","change","view"],
},
"Member": {
#django app model specific permissions
"project" : ["view"],
"order" : ["view"],
"staff time sheet" : ["add","delete","change","view"],
},
}
USERS = {
"my_member_user" : ["Member","member#domain.cu","1234*"],
"my_admin_user" : ["Administration","admin#domain.ca","1234"],
"Admin" : ["Administration","superuser#domain.cu","1234"],
}
class Command(BaseCommand):
help = "Creates read only default permission groups for users"
def handle(self, *args, **options):
for group_name in GROUPS:
new_group, created = Group.objects.get_or_create(name=group_name)
# Loop models in group
for app_model in GROUPS[group_name]:
# Loop permissions in group/model
for permission_name in GROUPS[group_name][app_model]:
# Generate permission name as Django would generate it
name = "Can {} {}".format(permission_name, app_model)
print("Creating {}".format(name))
try:
model_add_perm = Permission.objects.get(name=name)
except Permission.DoesNotExist:
logging.warning("Permission not found with name '{}'.".format(name))
continue
new_group.permissions.add(model_add_perm)
for user_name in USERS:
new_user = None
if user_name == "Admin":
new_user, created = User.objects.get_or_create(username=user_name,is_staff = True,is_superuser = True, email = USERS[user_name][1])
else:
new_user, created = User.objects.get_or_create(username=user_name,is_staff = True, email = USERS[user_name][1])
new_user.set_password(USERS[user_name][2])
new_user.save()
if USERS[user_name][0] == str(new_group):
new_group.user_set.add(new_user)
print("Adding {} to {}".format(user_name,new_group))

Django: Merge databases for same app

I have the same django app running on two servers, each has its own local database. I want to migrate to a single server, again with a local database.
What's the easiest way to populate the new database with the models data of the old databases without losing references between models? (primary keys issues etc...)
there's always the dump data from django, which is pretty easy to use.
or you could do this manually:
if the 2 databases share the same data (they are mirror one to another) and the same table structure, you could just run a syncdb from django to create the new table structure and then dump and import (i'm assuming you're using mysql, but the general idea is the same) the old database into the new one
if the two databases share different data (still with the same structure) you should import every single row of the two databases: this way, you'll keep relations etc, but you'll have your unique id updated to the new sole db.
if the two databases are different in both data and structure, you'll have to run two sincdb and two imports, but this doesn't seem to be your case
This is a reference to MySQL dump command
If your databases share the same data model with different objects, you could use this custom command I created for a similar problem.
Instead of merging fixture data with your existing models (as it does loaddata) it appends all fixtures object by resetting all pk.
M2M relations are managed at the end of the process, mapping the old primary keys with the new primary keys:
import os
import warnings
from collections import defaultdict
from django.core.management import CommandError
from django.core.management.utils import parse_apps_and_model_labels
from django.core.management.commands.loaddata import Command as LoadDataCommand, humanize
from django.core.management.color import no_style
from django.db import (
DEFAULT_DB_ALIAS, DatabaseError, IntegrityError, connections, router
)
from django.core import serializers
from django.db import transaction
from django.db.models.fields.related import RelatedField, ManyToManyField
class Command(LoadDataCommand):
help = 'Installs the named fixture(s) in the database.'
missing_args_message = (
"No database fixture specified. Please provide the path of at least "
"one fixture in the command line."
)
def add_arguments(self, parser):
parser.add_argument('args', metavar='fixture', nargs='+', help='Fixture labels.')
parser.add_argument(
'--database', default=DEFAULT_DB_ALIAS,
help='Nominates a specific database to load fixtures into. Defaults to the "default" database.',
)
parser.add_argument(
'--app', dest='app_label',
help='Only look for fixtures in the specified app.',
)
parser.add_argument(
'-e', '--exclude', action='append', default=[],
help='An app_label or app_label.ModelName to exclude. Can be used multiple times.',
)
parser.add_argument(
'--format',
help='Format of serialized data when reading from stdin.',
)
# TODO delete equals to overridden
def handle(self, *fixture_labels, **options):
self.using = options['database']
self.app_label = options['app_label']
self.verbosity = options['verbosity']
self.excluded_models, self.excluded_apps = parse_apps_and_model_labels(options['exclude'])
self.format = options['format']
with transaction.atomic(using=self.using):
self.appenddata(fixture_labels)
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(self.using):
connections[self.using].close()
def appenddata(self, fixture_labels):
# Most of the code is used only to manage transaction and fixture file format reuser it and override load_label instead
self.loaddata(fixture_labels)
def load_label(self, fixture_label):
"""Load fixtures files for a given label."""
self.objs_idx = ObjectDict()
self.objects = []
self.show_progress = self.verbosity >= 3
self.deferred_m2m = []
for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label):
_, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
open_method, mode = self.compression_formats[cmp_fmt]
fixture = open_method(fixture_file, mode)
try:
self.fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
if self.verbosity >= 2:
self.stdout.write(
"Installing %s fixture '%s' from %s."
% (ser_fmt, fixture_name, humanize(fixture_dir))
)
objects = serializers.deserialize(
ser_fmt, fixture, using=self.using, ignorenonexistent=True,
handle_forward_references=False,
)
for obj in objects:
objects_in_fixture += 1
if (obj.object._meta.app_config in self.excluded_apps or
type(obj.object) in self.excluded_models):
continue
if router.allow_migrate_model(self.using, obj.object.__class__):
loaded_objects_in_fixture += 1
self.models.add(obj.object.__class__)
# Load all fixture in memory
self.objs_idx.append_deserialized_object(obj)
self.objects.append(obj)
if obj.deferred_fields:
self.objs_with_deferred_fields.append(obj)
if objects and self.show_progress:
self.stdout.write('') # add a newline after progress indicator
self.loaded_object_count += loaded_objects_in_fixture
self.fixture_object_count += objects_in_fixture
except Exception as e:
if not isinstance(e, CommandError):
e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),)
raise e
finally:
fixture.close()
# Warn if the fixture we loaded contains 0 objects.
if objects_in_fixture == 0:
warnings.warn(
"No fixture data found for '%s'. (File format may be "
"invalid.)" % fixture_name,
RuntimeWarning
)
# Once you have all object in memory you can load them
for obj in self.objects:
self.process_object(obj.object)
if self.verbosity >= 1:
self.stdout.write('... All objects saved ...')
# Once all objects have been save (append mode) and new pks have been assigned add m2m relations
for obj, field_attname, related_pk in self.deferred_m2m:
attr = getattr(obj, field_attname)
attr.add(related_pk)
if self.verbosity >= 3:
self.stdout.write('Adding relation for field {0}: {1} -> {2}'.format(field_attname, obj.pk, related_pk))
# Disabled for security reason
# raise ValueError('Disabled')
def process_object(self, obj):
if obj is None:
raise ValueError('None object in process object')
old_pk = obj.pk
new_pk = self.objs_idx[obj]['new_pk']
# Object has been save yet no work
if new_pk:
return new_pk
self.manage_related_field(obj)
if self.verbosity >= 2:
self.stdout.write('Saving object: (%s, %s)' % (obj.__class__, obj))
obj.pk = None
try:
obj.save(using=self.using)
if self.show_progress:
self.stdout.write(
'\rSaving object: (%s, %s)' % (obj.__class__, obj),
ending=''
)
# psycopg2 raises ValueError if data contains NUL chars.
except (DatabaseError, IntegrityError, ValueError) as e:
e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
'app_label': obj.object._meta.app_label,
'object_name': obj.object._meta.object_name,
'pk': obj.object.pk,
'error_msg': e,
},)
raise
self.objs_idx.data[obj._meta.model][old_pk]['new_pk'] = obj.pk
return obj.pk
def manage_related_field(self, obj):
related_fields = [field for field in obj._meta.get_fields() if isinstance(field, RelatedField)]
if len(related_fields) > 0: # has not related field
for field in related_fields:
if field.related_model in self.excluded_models:
continue
if type(field) is ManyToManyField:
attr = getattr(obj, field.attname)
attr.clear()
m2m_pks = self.objs_idx[obj]['deserialized_object'].m2m_data[field.name]
for m2m_pk in m2m_pks:
related_obj = self.objs_idx.data[field.related_model][m2m_pk]['object']
new_related_pk = self.process_object(related_obj)
self.deferred_m2m.append((obj, field.attname, new_related_pk))
# attr.add(new_related_pk)
else:
related_obj = self.objs_idx.data[field.related_model][getattr(obj, field.attname)]['object']
if related_obj is not None:
new_related_pk = self.process_object(related_obj)
setattr(obj, field.attname, new_related_pk)
class ObjectDict(object):
"""
Dictionary to easily retrieve fixture object based on class and their original primary key
"""
def __init__(self):
self.data = defaultdict(lambda: defaultdict(lambda: {'new_pk': None, 'object': None}))
#staticmethod
def from_deserialized_objects(deserialized_objects):
instance = ObjectDict()
for deserialized_object in deserialized_objects:
instance.append_deserialized_object(deserialized_object)
return instance
def __getitem__(self, item):
return self.data[item._meta.model][item.old_pk]
def append_deserialized_object(self, deserialized_object):
obj = deserialized_object.object
setattr(obj, 'old_pk', obj.pk)
self[obj]['object'] = obj
self[obj]['deserialized_object'] = deserialized_object
I suggest you to test everything in the django default test DB, with this TestCase.
It uses a pre dumped fixture file from the django app (using python manage.py dumpdata) to populate the test DB, after that it applies the custom command to append all object from other database fixture dumps.
from collections import defaultdict
from django.core import serializers
from django.core.management.utils import parse_apps_and_model_labels
from django.test import Client, TestCase
from utils.tests import reverse
from django.core.management import call_command
from django.test import TestCase
from django.apps import apps
from apps.commons.accounts.models import User
from apps.commons.accounts.tests import MultiUserTestCase
class TestCustomCommands(TestCase):
# Create fixture from the primary database to test everything
fixtures = ['tmp/dump/test_append_data_fixtures_pre.json']
def test_appenddata(self):
fixture_to_import = 'tmp/dump/fixtures_to_import.json'
excludes = ['sites.Site']
# Counts objects before appenddata per model
count_pre = {}
for model in apps.get_models():
count_pre[model] = model.objects.count()
self.excluded_models, self.excluded_apps = parse_apps_and_model_labels(excludes)
# Counts objects to append per model
with open(fixture_to_import, 'r') as f:
objects = serializers.deserialize('json', f, ignorenonexistent=True)
count_new = defaultdict(lambda: 0)
for obj in objects:
if obj.object._meta.model in excludes:
continue
count_new[obj.object._meta.model] += 1
command = ['appenddata', fixture_to_import]
for exclude in excludes:
command += ['-e', exclude]
command += ['-v', '0']
call_command(*command)
# Verify with count that all objects have been imported
for model in apps.get_models():
self.assertEqual(count_pre[model] + count_new[model], model.objects.count(), msg='Count mismatch for model %s' % model)
Example of test (appending data from Website2 into Website1):
# Website 1
python manage.py dumpdata app1 app2 ... > test_append_data_fixtures_pre.json
# Website 2
python manage.py dumpdata app1 app2 ... > fixture_to_import.json
# Website 1, run the provided test
python manage.py test TestCustomCommands.test_appenddata
Example of usage (appending data from Website2 into Website1):
# Website 1
python manage.py appenddata fixture_to_import.json