How to use UUIDField for SQLite? - django

How can I generate a UUIDField that works for SQLite?
I want to use SQLite instead of Postgres for my tests so they run faster.
# settings.py
DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
# ...
}
}
# Tests use sqlite instead of postgres
import sys
if (
"test" in sys.argv or "test_coverage" in sys.argv
): # Covers regular testing and django-coverage
DATABASES["default"]["ENGINE"] = "django.db.backends.sqlite3"
However, I don't seem to be able to create a UUID that fits Django's UUIDField for SQLite:
A field for storing universally unique identifiers. Uses Python’s UUID class. When used on PostgreSQL, this stores in a uuid datatype, otherwise in a char(32).
The following doesn't work even though the uuid value is 32 chars:
# models.py
class Item(models.Model):
uuid = models.UUIDField()
# tests.py
uuid = str(uuid.uuid4()).replace("-", "")
Item.objects.create(uuid=uuid)
I get this error: django.db.utils.InterfaceError: Error binding parameter 4 - probably unsupported type.
Edit:
Here is the full error:
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.9/site-packages/django/db/backends/sqlite3/base.py", line 423, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.InterfaceError: Error binding parameter 4 - probably unsupported type.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/app/library/tests/tests_views_auth.py", line 423, in test_post_list_items_works
c1 = Chapter.objects.create(title="B1C1", body="B1C1", parent=b1)
File "/24reads/library/models.py", line 117, in create
return super().create(**kwargs)
File "/usr/local/lib/python3.9/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/django/db/models/query.py", line 453, in create
obj.save(force_insert=True, using=self.db)
File "/usr/local/lib/python3.9/site-packages/django_lifecycle/mixins.py", line 134, in save
save(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/mptt/models.py", line 1091, in save
self.insert_at(
File "/usr/local/lib/python3.9/site-packages/mptt/models.py", line 771, in insert_at
self._tree_manager.insert_node(
File "/usr/local/lib/python3.9/site-packages/mptt/managers.py", line 42, in wrapped
return getattr(self._base_manager, method.__name__)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/mptt/managers.py", line 43, in wrapped
return method(self, *args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/mptt/managers.py", line 537, in insert_node
self._create_space(2, space_target, tree_id)
File "/usr/local/lib/python3.9/site-packages/mptt/managers.py", line 816, in _create_space
self._manage_space(size, target, tree_id)
File "/usr/local/lib/python3.9/site-packages/mptt/managers.py", line 1052, in _manage_space
cursor.execute(
File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.9/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.9/site-packages/django/db/backends/sqlite3/base.py", line 423, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.InterfaceError: Error binding parameter 4 - probably unsupported type.
----------------------------------------------------------------------
My specific use case sees me override the base MPTTModel and TreeManager to use a UUID instead of a continuguous integer to avoid concurrency issues as outlined here: https://github.com/django-mptt/django-mptt/issues/555#issuecomment-331315715
class AsyncSafeTreeManager(TreeManager):
def _get_next_tree_id(self):
if (
"test" in sys.argv or "test_coverage" in sys.argv
): # Covers regular testing and django-coverage
return uuid.uuid4()
return generate_ulid_as_uuid()
class AsyncSafeMPTTModel(MPTTModel):
objects = AsyncSafeTreeManager()
tree_id = models.UUIDField()
class Meta:
abstract = True
This works fine when I'm using PostGRE but does not work with SQLite.

Regardless what the underlying type is, Django will transform a UUID to the correct format, and insert it in the database. For SQLite that thus means that the UUIDField will tranform it to a string.
You thus can create an item with:
import uuid
Item.objects.create(
uuid=uuid.uuid4()
)

Related

Django POST http://127.0.0.1:8000/create_member/ 500 (Internal Server Error)

i try to add usernames to my video chatting app and this error occured. I try to fetch create_member but this error occured. I try to solve many times but failed. Kindly Check it.
Uncaught (in promise) SyntaxError: Unexpected token. This error occured in console and i can't get my video and audio tracks.
Internal Server Error: /create_member/
Traceback (most recent call last):
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\utils.py", line
89, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\sqlite3\base.py", line 477, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: no such column: videoApp_roommember.name
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\core\handlers\exception.py", line 55, in inner
response = get_response(request)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\core\handlers\base.py", line 197, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\views\decorators\csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "C:\Users\hp\Desktop\ChattingApp\VideoChatting\videoApp\views.py", line 76, in createMember
member, created = RoomMember.objects.get_or_create(
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\models\manager.py", line
85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\models\query.py", line 657, in get_or_create
return self.get(**kwargs), False
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\models\query.py", line 492, in get
num = len(clone)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\models\query.py", line 302, in __len__
self._fetch_all()
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\models\query.py", line 1507, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\models\query.py", line 57, in __iter__
results = compiler.execute_sql(
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\models\sql\compiler.py",
line 1361, in execute_sql
cursor.execute(sql, params)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\utils.py", line
103, in execute
return super().execute(sql, params)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\utils.py", line
67, in execute
return self._execute_with_wrappers(
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\utils.py", line
80, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\utils.py", line
84, in _execute
with self.db.wrap_database_errors:
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\utils.py", line 91, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\utils.py", line
89, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\hp\AppData\Local\Programs\Python\Python310\lib\site-packages\django\db\backends\sqlite3\base.py", line 477, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: no such column: videoApp_roommember.name
[11/Oct/2022 07:47:37] "POST /create_member/ HTTP/1.1" 500 141222
So, here is my views.py file:
#csrf_exempt
def createMember(request):
data = json.loads(request.body)
member, created = RoomMember.objects.get_or_create(
name=data['name'],
uid=data['UID'],
room_name=data['room_name']
)
return JsonResponse({'name':data['name']}, safe=False)
here is my script.js file
let createMember = async () => {
let response = await fetch('/create_member/', {
method:'POST',
headers: {
'Content-Type':'application/json'
},
body:JSON.stringify({'name':NAME, 'room_name':CHANNEL, 'UID':UID})
})
let member = await response.json()
return member
}
And here is models.py file
class RoomMember(models.Model):
name = models.CharField(max_length=200)
uid = models.CharField(max_length=1000)
room_name = models.CharField(max_length=200)
insession = models.BooleanField(default=True)
def __str__(self):
return self.name
In django's docs under migrations it says:
Migrations are Django’s way of propagating changes you make to your models (adding a field, deleting a model, etc.) into your database schema.
So the error says
django.db.utils.OperationalError: no such column: videoApp_roommember.name
Which means the changes you made, which is adding name to the model, has not been migrated to take effect on the database table so whenever you make a query or try to create a room member the error above will be thrown because of not finding the name column in the DB.
So you need to:
Run python manage.py makemigrations to create the migrations file for the changes you made.
Run python manage.py migrate to make the updates to the database

Django doesn't create test database when running test

I read in Django document that a blank database will be created for testing. I am new to Django so I barely changed anything in setting.py and currently using Sqlite database. However, when I run python manage.py test, Django keep telling the user has been existed, so I tried changing the username for the created users in TestCase, run the test again and found out that the new users are created in the existing database. The test file is as bellow:
class UserTestCase(unittest.TestCase):
def setUp(self):
admin = User.objects.create(username="admin", password="1")
user1 = User.objects.create(username="user1", password="1")
user2 = User.objects.create(username="user2", password="1")
admin.following.add(user1)
admin.followers.add(user2)
def test_users_count(self):
self.assertEqual(User.objects.count()==3)
My model is as bellow:
class User(AbstractUser):
followers = models.ManyToManyField('self', related_name="following", symmetrical=False, through='Follow', through_fields=('followee', 'follower'))
def __str__(self):
return f"{self.username}"
def serialize(self):
return {
"id": self.id,
"username": self.username,
}
class Follow(models.Model):
followee = models.ForeignKey(
'User',
on_delete=models.CASCADE,
related_name='+'
)
follower = models.ForeignKey(
'User',
on_delete=models.CASCADE,
related_name='+'
)
def clean(self, *args, **kwargs):
if self.follower__id == self.followee__id:
raise ValidationError('Can not follow self.')
return super().clean(*args, **kwargs)
class Meta:
constraints = [
models.UniqueConstraint(fields=['follower', 'followee'], name='follow_once'),
models.CheckConstraint(check=~Q(follower=F('followee')), name='not_follow_self')
]
EDIT: Bellow is the output when I run python manage.py test
(env_web) PS C:\Users\HL94NVT\Programming\web_development\project4> python manage.py test
System check identified no issues (0 silenced).
EE
======================================================================
ERROR: test_following_count (network.tests.UserTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\sqlite3\base.py", line 413, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.IntegrityError: UNIQUE constraint failed: network_user.username
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\HL94NVT\Programming\web_development\project4\network\tests.py", line 21, in setUp
admin = User.objects.create(username="admin1", password="1")
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\query.py", line 447, in create
obj.save(force_insert=True, using=self.db)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\contrib\auth\base_user.py", line 67, in save
super().save(*args, **kwargs)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 754, in save
force_update=force_update, update_fields=update_fields)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 792, in save_base
force_update, using, update_fields,
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 895, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 935, in _do_insert
using=using, raw=raw,
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\query.py", line 1254, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\sql\compiler.py", line 1397, in execute_sql
cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\sqlite3\base.py", line 413, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.IntegrityError: UNIQUE constraint failed: network_user.username
======================================================================
ERROR: test_users_count (network.tests.UserTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\sqlite3\base.py", line 413, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.IntegrityError: UNIQUE constraint failed: network_user.username
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\HL94NVT\Programming\web_development\project4\network\tests.py", line 21, in setUp
admin = User.objects.create(username="admin1", password="1")
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\query.py", line 447, in create
obj.save(force_insert=True, using=self.db)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\contrib\auth\base_user.py", line 67, in save
super().save(*args, **kwargs)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 754, in save
force_update=force_update, update_fields=update_fields)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 792, in save_base
force_update, using, update_fields,
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 895, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\base.py", line 935, in _do_insert
using=using, raw=raw,
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\query.py", line 1254, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\models\sql\compiler.py", line 1397, in execute_sql
cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\HL94NVT\Programming\web_development\env_web\lib\site-packages\django\db\backends\sqlite3\base.py", line 413, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.IntegrityError: UNIQUE constraint failed: network_user.username
----------------------------------------------------------------------
Ran 2 tests in 0.063s
Using django.test.TestCase instead of unittest.TestCase helps according to the document
https://docs.djangoproject.com/en/3.1/topics/testing/overview/
If your tests rely on database access such as creating or querying models, be sure to create your test classes as subclasses of django.test.TestCase rather than unittest.TestCase.
Using unittest.TestCase avoids the cost of running each test in a transaction and flushing the database, but if your tests interact with the database their behavior will vary based on the order that the test runner executes them. This can lead to unit tests that pass when run in isolation but fail when run in a suite.

Can't insert data, Django primary key doesn't have a default

In Django 3.0, I have a set of Models consisting of an abstract BaseData, a Model Data that extends it, and a Model of underlying data FKData that Data foreign-keys to:
# models.py
from django.db import models
class BaseData(models.Model):
class Meta:
abstract = True
class Data(BaseData):
data = models.ForeignKey(
FKData,
on_delete=models.CASCADE,
blank=True,
null=True,
default=None
)
class FKData(models.Model):
text = models.CharField(
help_text='underlying data'
)
The error occurs when I try to INSERT into Data. For example,
mysql> INSERT INTO appname_data (data_id) SELECT data_id FROM appname_othertable
ERROR 1364 (HY000): Field 'basedata_ptr_id' doesn't have a default value
basedata_ptr_id is a Django-generated primary key field that Data has due to being a subclass of BaseData:
mysql> DESCRIBE appname_data;
+-----------------+---------+------+-----+---------+-------+
| Field | Type | Null | Key | Default | Extra |
+-----------------+---------+------+-----+---------+-------+
| basedata_ptr_id | int(11) | NO | PRI | NULL | |
| data_id | int(11) | YES | MUL | NULL | |
+-----------------+---------+------+-----+---------+-------+
2 rows in set (0.00 sec)
The error sounds like it isn't being set to AUTO INCREMENT the way normal primary keys are?
If I try filling the Data table using the Django shell, I get the same error but with a bigger stack trace that might be useful:
Traceback (most recent call last):
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/mysql/base.py", line 74, in execute
return self.cursor.execute(query, args)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/MySQLdb/cursors.py", line 209, in execute
res = self._query(query)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/MySQLdb/cursors.py", line 315, in _query
db.query(q)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/MySQLdb/connections.py", line 239, in query
_mysql.connection.query(self, query)
MySQLdb._exceptions.IntegrityError: (1364, "Field 'basedata_ptr_id' doesn't have a default value")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<console>", line 6, in <module>
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/models/base.py", line 746, in save
force_update=force_update, update_fields=update_fields)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/models/base.py", line 784, in save_base
force_update, using, update_fields,
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/models/base.py", line 887, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/models/base.py", line 926, in _do_insert
using=using, raw=raw,
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/models/query.py", line 1204, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1391, in execute_sql
cursor.execute(sql, params)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/utils.py", line 100, in execute
return super().execute(sql, params)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/django/db/backends/mysql/base.py", line 74, in execute
return self.cursor.execute(query, args)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/MySQLdb/cursors.py", line 209, in execute
res = self._query(query)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/MySQLdb/cursors.py", line 315, in _query
db.query(q)
File "/home/username/ProjectName/env/lib/python3.6/site-packages/MySQLdb/connections.py", line 239, in query
_mysql.connection.query(self, query)
django.db.utils.IntegrityError: (1364, "Field 'basedata_ptr_id' doesn't have a default value")
What went wrong, and how do I fix it?
Edit: I've decided to wipe my database and bring all development to a halt while I spend a couple of weeks rebuilding it, all because I overlooked a single line, because I'm using "The framework for perfectionists with deadlines."
Nevertheless, this question deserves a better answer. Based on the comments, I suspect the best way to approach the answer is the following:
Consider two scenarios.
Scenario A
I define BaseData as an abstract class
class BaseData(models.Model):
class Meta:
abstract = True
I run makemigrations to create a migration file
Scenario B
I define BaseData as a non-abstract class
class BaseData(models.Model):
class Meta:
pass
I run makemigrations to create a migration file.
I alter BaseData to be an abstract class
class BaseData(models.Model):
class Meta:
abstract = True
I run makemigrations to create a second migration file.
In principle, the combination of the two migrations from Scenario B should exactly equal the migration from Scenario A. In practice, something is different. The first step in answering this question is determining what's different about the migration directives between the two scenarios. Then, we can formulate a process of manually editing the migration files to solve the problem without wiping the database, for people in the future who have the same problem.

Django Import Export - UNIQUE constraint failed

I am following this documentation on how to use django-import-export:
https://django-import-export.readthedocs.io/en/latest/getting_started.html#declaring-fields
I have an excel sheet that looks like below
I want to store the data in this model:
class ExcelData(models.Model):
var1 = models.CharField(max_length=200)
var2 = models.CharField(max_length=200,unique=True)
var3 = models.CharField(max_length=200)
var4 = models.CharField(max_length=200)
This is how far I got:
#admin.register(ExcelData)
class ViewAdmin(ImportExportModelAdmin):
exclude = ('id',)
class ExcelDataResource(resources.ModelResource):
var1 = Field(attribute='var1', column_name='Name')
var2 = Field(attribute='var2', column_name='SAP_ID')
var3 = Field(attribute='var3', column_name='Abbreviation')
var4 = Field(attribute='var4', column_name='Max. Capa')
class Meta:
model = ExcelData
import_id_fields = ('var2',)
exclude = ('id',)
This is what I get:
Here is the CSV File:
http://www.sharecsv.com/s/9d1112392cd7f10378de7fc0811dd0c9/REAL_CSV_SIMPLE.csv
When I try to import multiple rows like this:
I get this error:
Line number: 2 - UNIQUE constraint failed: myapp_exceldata.var2
b, e, h, k
Traceback (most recent call last):
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\sqlite3\base.py", line 396, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.IntegrityError: UNIQUE constraint failed: myapp_exceldata.var2
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Users\...\env\lib\site-packages\import_export\resources.py", line 527, in import_row
self.save_instance(instance, using_transactions, dry_run)
File "D:\Users\...\env\lib\site-packages\import_export\resources.py", line 320, in save_instance
instance.save()
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 746, in save
force_update=force_update, update_fields=update_fields)
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 784, in save_base
force_update, using, update_fields,
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 887, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 926, in _do_insert
using=using, raw=raw,
File "D:\Users\...\env\lib\site-packages\django\db\models\manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "D:\Users\...\env\lib\site-packages\django\db\models\query.py", line 1204, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "D:\Users\...\env\lib\site-packages\django\db\models\sql\compiler.py", line 1384, in execute_sql
cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 100, in execute
return super().execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\sqlite3\base.py", line 396, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.IntegrityError: UNIQUE constraint failed: myapp_exceldata.var2
Line number: 3 - UNIQUE constraint failed: myapp_exceldata.var2
c, f, i, l
Traceback (most recent call last):
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\sqlite3\base.py", line 396, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.IntegrityError: UNIQUE constraint failed: myapp_exceldata.var2
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Users\...\env\lib\site-packages\import_export\resources.py", line 527, in import_row
self.save_instance(instance, using_transactions, dry_run)
File "D:\Users\...\env\lib\site-packages\import_export\resources.py", line 320, in save_instance
instance.save()
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 746, in save
force_update=force_update, update_fields=update_fields)
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 784, in save_base
force_update, using, update_fields,
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 887, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "D:\Users\...\env\lib\site-packages\django\db\models\base.py", line 926, in _do_insert
using=using, raw=raw,
File "D:\Users\...\env\lib\site-packages\django\db\models\manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "D:\Users\...\env\lib\site-packages\django\db\models\query.py", line 1204, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "D:\Users\...\env\lib\site-packages\django\db\models\sql\compiler.py", line 1384, in execute_sql
cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 100, in execute
return super().execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "D:\Users\...\env\lib\site-packages\django\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "D:\Users\...\env\lib\site-packages\django\db\backends\sqlite3\base.py", line 396, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.IntegrityError: UNIQUE constraint failed: myapp_exceldata.var2
Here is the CSV file:
http://www.sharecsv.com/s/3f09eb14f7916933604481fd999d03db/REAL_CSV_FULL.csv
Thank you for any suggestions
The way import works is the following:
You need a unique identifier so that when importing, the import-export module "knows" whether it needs to create a new row or modify an existing row.
If that unique identifier is not the model's id field (which in your case is the auto-incremented row number), then it must be some other unique field. None of the fields var1 ... var4 on your model are declared as unique, so as you've defined your model currently, it's impossible to import.
You can use multiple "id fields" if a combination of fields is unique together, but that's also not the case on your model.
The import module with actually do a get_or_create() call on your database, using the unique_id_fields as the get parameters. This should never return more than 1 row, if it were to return multiple rows, the import would crash.
To give you an idea, and assuming SAP_ID is a unique identifier (I doubt Max. Capacity is, the name doesn't indicate it would be), this should be your model:
class ExcelData(models.Model):
var1 = models.CharField(max_length=200)
var2 = models.CharField(max_length=200, unique=True)
var3 = models.CharField(max_length=200)
var4 = models.CharField(max_length=200)
Note your model also has the implicit field id, which import-export knows about and you have to decide what to do with it, since it's not in your import data. This is why you're receiving the error you mentioned, because it's a field in your model that you haven't assigned to anything. Best is to exclude it, since it's not relevant here (Django will auto-increment a new id if the row doesn't exist yet, like when you create the model with ExcelData.objects.create()):
class ExcelDataResource(resources.ModelResource):
var1 = Field(attribute='var1', column_name='Name')
var2 = Field(attribute='var2', column_name='SAP_ID')
var3 = Field(attribute='var3', column_name='Abbreviation')
var4 = Field(attribute='var4', column_name='Max. Capa')
class Meta:
model = ExcelData
import_id_fields = ('var2',)
exclude = ('id',)
With this code and the following csv file, it works, I've tested it:
Name,SAP_ID,Abbreviation,Max. Capa
a,d,g,j
b,e,h,k
c,f,i,l
When trying to import a second time or if there's already some data in the database where the value of var2 is d, e or f, you'll get UNIQUE constraint failed errors for exceldata.var2.
Note: If Max. Capa really is your unique id field, then you shouldn't assign it to var4 but to id. That would also work, although as I mentioned above I doubt that's your desired behaviour.
UPDATE/Note 2: There's a bug in django-import-export that will cause UNIQUE constraints exceptions when you try to update existing data: This happens if your unique_id_field has a different name than your column_name in the data you import (e.g. var2 for SAP_ID) and you try to re-import data which contains an already existing row with that same id (e.g. because you want to change the other values). It should update the row, but currently it throws an exception. This doesn't explain the exception on var4 (that's because you set var4 to be unique as well and if you import a different row with a duplicate var4 (Max. Capa) value, then you'll also get an exception).

PosGis and Django-Tenants

(Using the library django-tenants for tenant separated multi-tenancy) For PostGis support the docs say to add ORIGINAL_BACKEND = "django.contrib.gis.db.backends.postgis". I have this, however, when I go to create a new tenant I get the following error:
Traceback (most recent call last):
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\celery\app\trace.py", line 382, in trace_task
R = retval = fun(*args, **kwargs)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\celery\app\trace.py", line 641, in __protected_call__
return self.run(*args, **kwargs)
File "C:\Users\Cole\Documents\GitHub\Elevate-RA-Django-App\returns_app\apps\tenant_stores\tasks.py", line 28, in create_tenant_task
tenant.save()
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django_tenants\models.py", line 93, in save
self.create_schema(check_if_exists=True, verbosity=verbosity)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django_tenants\models.py", line 143, in create_schema
verbosity=verbosity)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\core\management\__init__.py", line 141, in call_command
return command.execute(*args, **defaults)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\core\management\base.py", line 335, in execute
output = self.handle(*args, **options)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django_tenants\management\commands\migrate_schemas.py", line 63, in handle
executor.run_migrations(tenants=tenants)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django_tenants\migration_executors\standard.py", line 15, in run_migrations
run_migrations(self.args, self.options, self.codename, schema_name, idx=idx, count=len(tenants))
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django_tenants\migration_executors\base.py", line 34, in run_migrations
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\core\management\base.py", line 335, in execute
output = self.handle(*args, **options)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\core\management\commands\migrate.py", line 77, in handle
connection.prepare_database()
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\contrib\gis\db\backends\postgis\base.py", line 26, in prepare_database
cursor.execute("CREATE EXTENSION IF NOT EXISTS postgis")
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\db\backends\utils.py", line 100, in execute
return super().execute(sql, params)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\db\backends\utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\db\backends\utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\db\backends\utils.py", line 85, in _execute
return self.cursor.execute(sql, params)
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\db\utils.py", line 89, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "c:\users\cole\appdata\local\programs\python\python36-32\lib\site-packages\django\db\backends\utils.py", line 83, in _execute
return self.cursor.execute(sql)
django.db.utils.ProgrammingError: relation "spatial_ref_sys" does not exist
The spatial_ref_sys table exists within my public schema. The django.contrib.gis app is in my shared apps.
Any ideas?
The issue seems to be cause by the default PostGis backend, specifically the call to prepare the database for migration, by explicitly setting the search path prior to calling CREATE EXTENSION IF NOT EXISTS postgis I was able to migrate/create a schema by creating a custom DB backend that overrides this behaviour:
from django.contrib.gis.db.backends.postgis.base import (
DatabaseWrapper as OriginalPostGisDatabaseWrapper,
)
from django_tenants.utils import get_public_schema_name
class DatabaseWrapper(OriginalPostGisDatabaseWrapper):
"""
This database wrapper explicitly sets the search path when preparing the database, as
multi-schema environments (like with Django-tenants) can cause issues with the PostGis
backend.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.PUBLIC_SCHEMA_NAME = get_public_schema_name()
def prepare_database(self):
# Check that postgis extension is installed.
with self.cursor() as cursor:
cursor.execute('SET search_path = %s', params=[self.PUBLIC_SCHEMA_NAME])
cursor.execute("CREATE EXTENSION IF NOT EXISTS postgis")
Then, set your ORIGINAL_BACKEND setting to the location of the above DB backend instead of the standard PostGis backend.