How can I avoid "database is locked" sqlite3 errors in django? - django

Question

Django version 1.11, sqlite3 version 3.11.
I'm using WAL mode and a long timeout:
from django.apps import AppConfig
from django.db.backends.signals import connection_created
class SQLite3Config(AppConfig):
name = 'sqlite3_config'
def ready(self):
connection_created.connect(configure_sqlite)
# noinspection PyUnusedLocal
def configure_sqlite(sender, connection, **_):
if connection.vendor == 'sqlite':
cursor = connection.cursor()
cursor.execute('PRAGMA journal_mode=WAL;')
cursor.execute('PRAGMA busy_timeout=5000;')
I want to retain sqlite3 and not move to mysql or postgres because the application is small and is installed by users on multiple servers.
I believe WAL should allow "concurrent" writes by serializing them. The "Database is locked" problem was observed when small bursts (half a dozen or so) were received together.
I can reproduce the problem in the shell with threads. The django model method simply sets a flag and saves the model:
def activate(self):
self.activate = True
self.save()
When I use threads I find it fails if I launch a few threads that attempt it at the same time. There is no wait so the timeout is not involved. The error occurs before the 5 second busy timeout has elapsed (in less than two seconds):
In [2]: [NGThread(notifier_group.id).start() for notifier_group in NotifierGroup.objects.all()[:2]]
Out[2]: [None, None]
In [3]: [NGThread(notifier_group.id).start() for notifier_group in NotifierGroup.objects.all()[:3]]
Out[3]: [None, None, None]
In [4]: [NGThread(notifier_group.id).start() for notifier_group in NotifierGroup.objects.all()[:4]]
Out[4]: [None, None, None, None]
In [5]: Exception in thread Thread-97:
Traceback (most recent call last):
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/sqlite3/base.py", line 328, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/home/paul/wk/cliosoft/sosadmin/scratch.py", line 41, in run
toggle_active(notifier_group)
File "/home/paul/wk/cliosoft/sosadmin/scratch.py", line 30, in toggle_active
model.activate()
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 67, in activate
self.save()
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 33, in save
self.verify()
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 46, in verify
self.create_notifier(base_spec, model_set, group_event_condition)
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 57, in create_notifier
notifier.users = self.users.all()
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 534, in __set__
manager.set(value)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 1004, in set
self.add(*new_objs)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 931, in add
self._add_items(self.source_field_name, self.target_field_name, *objs)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 1100, in _add_is
for obj_id in new_ids
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/query.py", line 442, in bulk_create
ids = self._batched_insert(objs_without_pk, fields, batch_size)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/query.py", line 1083, in _batched_insert
self._insert(item, fields=fields, using=self.db)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/query.py", line 1060, in _insert
return query.get_compiler(using=using).execute_sql(return_id)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/sql/compiler.py", line 1099, in execute_sql
cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 80, in execute
return super(CursorDebugWrapper, self).execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/utils/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/sqlite3/base.py", line 328, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database is locked
Exception in thread Thread-98:
Traceback (most recent call last):
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/sqlite3/base.py", line 328, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/home/paul/wk/cliosoft/sosadmin/scratch.py", line 41, in run
toggle_active(notifier_group)
File "/home/paul/wk/cliosoft/sosadmin/scratch.py", line 28, in toggle_active
model.deactivate()
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 72, in deactivate
self.save()
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 33, in save
self.verify()
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 46, in verify
self.create_notifier(base_spec, model_set, group_event_condition)
File "/home/paul/wk/cliosoft/sosadmin/notifications/models/notifier_group.py", line 57, in create_notifier
notifier.users = self.users.all()
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 534, in __set__
manager.set(value)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 1004, in set
self.add(*new_objs)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 931, in add
self._add_items(self.source_field_name, self.target_field_name, *objs)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 1100, in _add_is
for obj_id in new_ids
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/query.py", line 442, in bulk_create
ids = self._batched_insert(objs_without_pk, fields, batch_size)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/query.py", line 1083, in _batched_insert
self._insert(item, fields=fields, using=self.db)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/query.py", line 1060, in _insert
return query.get_compiler(using=using).execute_sql(return_id)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/models/sql/compiler.py", line 1099, in execute_sql
cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 80, in execute
return super(CursorDebugWrapper, self).execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/utils/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/paul/.virtualenvs/sosadmin/lib/python3.5/site-packages/django/db/backends/sqlite3/base.py", line 328, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database is locked

Answer 1

I can't see anything in the release notes but I've upgraded to Django 2.0 and this problem has now disappeared. With the WAL set up in sqlite3 its all working very well now.

Answer 2

As the system has scaled, this error has re-occurred. I'm required to use sqlite3 for reasons outside the scope of this problem.
The timeout option in Django seems to be ignored. I tried implementing it myself at the db cursor level with a wrapper around sqlite and had the same problem (at that level you can't back out far enough to abandon the transaction and try a new connection).
I'd have liked a context manager e.g. with db_locked_retries():... but it is not possible to iterate within the with context so I created a db_retry function thus:
import time
import logging
import random
from django.conf import settings
import django.db
logger = logging.getLogger('django.db.backends')
def db_retry(fn, timeout=None):
"""Call fn with no arguments. If OperationalError exception, make retries until timeout has passed"""
timeout = timeout or settings.DATABASES['default'].get('OPTIONS', dict()).get('timeout', 5)
now = time.time()
give_up_time = now + timeout
retries = 0
while now < give_up_time:
now = time.time()
try:
result = fn()
if retries:
logger.warning(f'db_retry: Succeeded after {retries} retries')
return result
except django.db.OperationalError as exception:
msg = str(exception)
if 'locked' in msg: # pragma: no cover
retries += 1
wait_time = random.uniform(1, timeout / 10)
logger.warning(f'db_retry: {msg}: Retrying after {wait_time} seconds')
django.db.close_old_connections()
time.sleep(wait_time)
else: # pragma: no cover
logger.exception(f'db_retry: {msg}: Giving up')
raise
As it is very easy to define the function to pass to this function within the model methods etc. this addresses the problem and successfully uses the timeout to retry the access. This is currently handling some hundreds of concurrent uses and retries once every thirty minutes or so in my particular use case.
I've only needed to 'wrap' model code that potentially makes several writes so far.