django celery asynchronous file saves to storage - django

Question

So I want to asynchronously conduct file saves to django file storage. I've tried to achieve this by overwriting the default File System Storage class.
import hashlib
import os
import uuid
import django.core.files.storage as storage
from celery import shared_task
from django.core.files import File
class DefaultStorage(storage.FileSystemStorage):
def __init__(self):
super(DefaultStorage, self).__init__()
#shared_task
def _save(self, name, content):
# need to encrypt file before saving
return super(DefaultStorage, self)._save(name, content)
#shared_task
def _open(self, name, mode='rb'):
# need to decrypt before opening
return File(open(self.path(name), mode))
def get_available_name(self, name, max_length=None):
# we return a hash of the file given,
# in case we miss out on uniqueness, django calls
# the get_alternative_name method
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
file_root = hashlib.md5(file_root.encode()).hexdigest()
name = os.path.join(dir_name, file_root + file_ext)
return super(DefaultStorage, self).get_available_name(name, max_length)
def get_alternative_name(self, file_root, file_ext):
# we insert a random uuid hex string into the given
# file name before returning the same back
return '%s%s%s' % (file_root, uuid.uuid4().hex, file_ext)
def save(self, name, content, max_length=None):
# override existing system storage save method
# to ensure that we call the shared task and
# utilize the distributed task queue celery offers
has_chunks = hasattr(content, 'chunks')
name = content.name if not name else name
content = File(content, name) if not has_chunks else content
name = self.get_available_name(name, max_length=max_length)
saved = self._save.delay(self, name, content)
return saved
def open(self, name, mode='rb'):
# override existing system storage open method
# to ensure that we call the shared task and
# utilize the distributed task queue celery offers
return self._open.delay(name, mode)
The idea is to override the existing save method to fire off the shared_task named _save. However, I am facing the following error while trying to do the same.
Traceback (most recent call last):
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\views\decorators\csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\views\generic\base.py", line 70, in view
return self.dispatch(request, *args, **kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\views.py", line 509, in dispatch
response = self.handle_exception(exc)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\views.py", line 469, in handle_exception
self.raise_uncaught_exception(exc)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\views.py", line 480, in raise_uncaught_exception
raise exc
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\views.py", line 506, in dispatch
response = handler(request, *args, **kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\generics.py", line 242, in post
return self.create(request, *args, **kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\mixins.py", line 19, in create
self.perform_create(serializer)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\mixins.py", line 24, in perform_create
serializer.save()
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\serializers.py", line 205, in save
self.instance = self.create(validated_data)
File "C:\Users\iyapp\PycharmProjects\cloud-server\api_backend\serializers\partial.py", line 32, in create
return super(PartialUploadSerializer, self).create(validated_data)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\rest_framework\serializers.py", line 939, in create
instance = ModelClass._default_manager.create(**validated_data)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\query.py", line 447, in create
obj.save(force_insert=True, using=self.db)
File "C:\Users\iyapp\PycharmProjects\cloud-server\api_backend\models\uploads.py", line 30, in save
super(Upload, self).save(*args, **kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\base.py", line 754, in save
force_update=force_update, update_fields=update_fields)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\base.py", line 792, in save_base
force_update, using, update_fields,
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\base.py", line 895, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\base.py", line 935, in _do_insert
using=using, raw=raw,
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\query.py", line 1254, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1396, in execute_sql
for sql, params in self.as_sql():
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1341, in as_sql
for obj in self.query.objs
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1341, in <listcomp>
for obj in self.query.objs
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1340, in <listcomp>
[self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields]
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1291, in pre_save_val
return field.pre_save(obj, add=True)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\fields\files.py", line 307, in pre_save
file.save(file.name, file.file, save=False)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\django\db\models\fields\files.py", line 87, in save
self.name = self.storage.save(name, content, max_length=self.field.max_length)
File "C:\Users\iyapp\PycharmProjects\cloud-server\api_backend\models\storage\default_storage.py", line 48, in save
saved = self._save.delay(self, name, content)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\celery\app\task.py", line 421, in delay
return self.apply_async(args, kwargs)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\celery\app\task.py", line 565, in apply_async
**options
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\celery\app\base.py", line 749, in send_task
amqp.send_task_message(P, name, message, **options)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\celery\app\amqp.py", line 532, in send_task_message
**properties
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\messaging.py", line 166, in publish
compression, headers)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\messaging.py", line 249, in _prepare
body) = dumps(body, serializer=serializer)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\serialization.py", line 220, in dumps
payload = encoder(data)
File "C:\Users\iyapp\AppData\Local\Programs\Python\Python36-32\lib\contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\serialization.py", line 53, in _reraise_errors
reraise(wrapper, wrapper(exc), sys.exc_info()[2])
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\exceptions.py", line 21, in reraise
raise value.with_traceback(tb)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\serialization.py", line 49, in _reraise_errors
yield
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\serialization.py", line 220, in dumps
payload = encoder(data)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\utils\json.py", line 66, in dumps
**dict(default_kwargs, **kwargs))
File "C:\Users\iyapp\AppData\Local\Programs\Python\Python36-32\lib\json\__init__.py", line 238, in dumps
**kw).encode(obj)
File "C:\Users\iyapp\AppData\Local\Programs\Python\Python36-32\lib\json\encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "C:\Users\iyapp\AppData\Local\Programs\Python\Python36-32\lib\json\encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "C:\Users\iyapp\PycharmProjects\cloud-server\venv\lib\site-packages\kombu\utils\json.py", line 55, in default
return super().default(o)
File "C:\Users\iyapp\AppData\Local\Programs\Python\Python36-32\lib\json\encoder.py", line 180, in default
o.__class__.__name__)
kombu.exceptions.EncodeError: Object of type 'DefaultStorage' is not JSON serializable
I believe I have found the source of the issue. It is because we cannot pass "file" objects as parameters to shared_task functions. If this is the case, then how can I send my file to the _save function?
Before sending, the file isn't saved yet. We need to save the file only after receiving.