On our project, with numerous requests to api in a short time, they are processed in different threads. When a user requests the creation of several instances at once in a table that is inherited from MPTT, this occurs in such a way that the front-end sends several different requests to the back-end in a very short time, each of which calls the code for creating one instance of this table. These requests are processed in different threads. But an unpleasant situation arises, when processing in different threads, instances are assigned the same tree and the same left and right numbers for different table rows. As a result, it turns out that the tree has in the same node several different values for this node.
Please help me to fix this problem. How we can deal with this problem?
models.py:
import sqlalchemy as sa
from sqlalchemy.orm import relationship
from sqlalchemy_mptt import BaseNestedSets
class UserFile(BaseNestedSets):
__tablename__ = "user_files"
id = sa.Column(sa.Integer, primary_key=True)
is_file: bool = sa.Column(sa.Boolean, default=False)
name: str = sa.Column(sa.String(100))
real_name: Optional[str] = sa.Column(sa.String(50))
size: int = sa.Column(sa.Integer, default=0)
downloaded_at: datetime = sa.Column(sa.DateTime, default=datetime.utcnow, nullable=False)
author_id = sa.Column("author_id", sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=True)
author = relationship("User", foreign_keys=[self.author_id])
def __repr__(self) -> str:
return self.name
schemas.py:
#spec_definition
class UserFileSchema(ExtendModelSchema):
class Meta:
model = UserFile
fields = ["id", "name", "is_file", "size", "downloaded_at"]
views.py:
#api.resource("/user/<int:user_id>/files/root/add_file")
class UserFilesAddRootFileResource(SchemaResource):
def post(self, user_id: int) -> dict:
content = request.files.get("file")
item = add_file(user_id, content, filename=content.filename)
return UserFileSchema().dump(item)
user_files.py:
def add_file(user_id: int, content: FileStorage, filename: str) -> UserFile:
size = file_size(content)
final_name = storage.save(None, content, True)
return add_user_file(
user_id=user_id,
name=filename,
is_file=True,
real_name=final_name,
size=size,
)
def add_user_file(
user_id: int,
name: str,
real_name: Optional[str] = None,
size: int = 0,
) -> UserFile:
base_name = name
if is_file:
base_name, ext = storage.splitext(base_name)
result_name = name
item_id = None
item = UserFile(
name=result_name,
real_name=real_name,
is_file=is_file,
size=size,
)
item.save()
return item
I am omitting the definition of some functions from the storage module, which are currently just a layer for working with the os.path.splitext(name) function or writing a file to disk, and they are not relevant to this problem.
I also wanted to ask why some tree nodes can have negative right and left numbers?
We use in our project Flask, SQLAlchemy, sqlalchemy-mptt==0.2.5
Related
I have 3 models: User, UserProfile and Programme. User holds the names of all users;
class User(AbstractUser):
first_name = model.Charfield(...)
last_name = model.Charfield(...)
...
UserProfile contains the attributes associated with each user (e.g. sales manager(sm), program manager(pm), delivery manager(dm) etc.
class UserProfile(model.Model):
user = model.ForeignKey(User, on_delete...)
sm = model.BooleanField(default = False, editable=True...)
pm = model.BooleanField(default = False, editable=True...)
dm = model.BooleanField(default = False, editable=True...)
A User may have any combination of UserProfile attributes. For example:
User-1: sm
User-2: sm, pm
User-3: dm
User-4: sm, dm
...
Every programme in Programme must have a user assigned for each of sm, pm, dm so my challenge is finding a way to generate a 'choices' list for each of sm, pm and dm limited to Users with the appropriate attribute.
class Programme(model.Models):
programme_name = models.Charfield(...)
AND ESSENTIALLY:
sm = only users with userprofile.sm = True
pm = only users with userprofile.pm = True
dm = only users with userprofile.dm = True
I've tried adding a function to UserProfile then referencing it in Programme:
UserProfile(model.Models):
....
def get_pm(self):
pm_list = []
pm = UserProfile.objects.filter(pm=True)
for i in pm:
pm_list.append(i.user.last_name)
return pm_list
and in Programme:
Programme(model.Models):
pm = models.Charfield(choices=UserProfile.get_pm, ...
but this generates the error: 'choices' must be an iterable (e.g., a list or tuple).
Similarly, I've tried adding the same function to Programmes/models.py but this generates the same error.
I know I'm missing the obvious but all help appreciated.
You dont need to create a separate model for User profile:
class User(AbstractUser):
first_name = model.Charfield(...)
last_name = model.Charfield(...)
POST_CHOICES = (
('sm', ("sm")),
('sm', ("pm")),
('sm', ("dm")),
)
post=models.CharField(choices=POST_CHOICES,max_length=150, blank=True)
programme_name = models.Charfield(max_length=150, blank=True)
You can filter out the users who are sm,for example using filter queries or you can use model managers for this
Well, after a ridiculous amount of reading, trial and error, a simple solution emerges: Simple override the form fields using an appropriate query -
class ProgrammeCreateForm(forms.ModelForm):
class Meta:
model = Programme
fields = [
'name', 'company', 'status', 'pm', 'sm', 'dm', 'idm','start_date', 'end_date', 'delivery_type'
]
def __init__(self, *args, **kwargs):
super(ProgrammeCreateForm, self).__init__(*args, **kwargs)
sm_choices = UserProfile.objects.filter(sm=True)
pm_choices = UserProfile.objects.filter(pm=True)
dm_choices = UserProfile.objects.filter(dm=True)
idm_choices = UserProfile.objects.filter(idm=True)
self.fields['sm'] = forms.ModelChoiceField(queryset=sm_choices)
self.fields['pm'] = forms.ModelChoiceField(queryset=pm_choices)
self.fields['dm'] = forms.ModelChoiceField(queryset=dm_choices)
self.fields['idm'] = forms.ModelChoiceField(queryset=idm_choices)
I want to write all types of complex queries,
for example :
If someone wants information "Fruit" is "Guava" in "Pune District" then they will get data for guava in pune district.
htt//api/?fruit=Guava&?district=Pune
If someone wants information "Fruit" is "Guava" in "Girnare Taluka" then they will get data for guava in girnare taluka.
htt://api/?fruit=Guava&?taluka=Girnare
If someone wants information for "Fruit" is "Guava" and "Banana" then they will get all data only for this two fruits, like wise
htt://api/?fruit=Guava&?Banana
But, when I run server then I cant get correct output
If i use http://api/?fruit=Banana then I get all data for fruit which is banana, pomegranate, guava instead of get data for fruit is only banana. So I am confuse what happen here.
can you please check my code, where I made mistake?
*Here is my all files
models.py
class Wbcis(models.Model):
Fruit = models.CharField(max_length=50)
District = models.CharField(max_length=50)
Taluka = models.CharField(max_length=50)
Revenue_circle = models.CharField(max_length=50)
Sum_Insured = models.FloatField()
Area = models.FloatField()
Farmer = models.IntegerField()
def get_wbcis(fruit=None, district=None, talkua=None, revenue_circle=None, sum_insured=None, area=None,min_farmer=None, max_farmer=None, limit=100):
query = Wbcis.objects.all()
if fuit is not None:
query = query.filter(Fruit=fruit)
if district is not None:
query = query.filter(District=district)
if taluka is not None:
query = query.filter(Taluka=taluka)
if revenue_circle is not None:
query = query.filter(Revenue_circle= revenue_circle)
if sum_insured is not None:
query = query.filter(Sum_Insured=sum_Insured)
if area is not None:
query = query.filter(Area=area)
if min_farmer is not None:
query = query.filter(Farmer__gte=min_farmer)
if max_farmer is not None:
query = query.filter(Farmer__lt=max_farmer)
return query[:limit]
Views.py
class WbcisViewSet(ModelViewSet):
queryset = Wbcis.objects.all()
serializer_class = WbcisSerializer
def wbcis_view(request):
fruit = request.GET.get("fruit")
district = request.GET.get("district")
taluka = request.GET.get("taluka")
revenue_circle = request.GET.get("revenue_circle")
sum_insured = request.GET.get("sum_insured")
area = request.GET.get("area")
min_farmer = request.GET.get("min_farmer")
max_farmer = request.GET.get("max_farmer")
wbcis = get_wbcis(fruit, district, taluka,revenue_circle,sum_insured,area, min_farmer, max_farmer)
#convert them to JSON:
dicts = []
for wbci in wbcis:
dicts.append(model_to_dict(wbci))
return JsonResponse(dicts)
Serializers.py
from rest_framework.serializers import ModelSerializer
from WBCIS.models import Wbcis
class WbcisSerializer(ModelSerializer):
class Meta:
model = Wbcis
fields=('id','Fruit','District','Sum_Insured','Area','Farmer','Taluka','Revenue_circle',)
whats need changes in this code for call these queries to get exact output?
I don't think that you're actually calling that view, judging by your usage I presume you're calling the viewset itself and then ignoring the query params.
You should follow the drf docs for filtering but essentially, provide the get queryset method to your viewset and include the code you currently have in your view in that
class WbcisViewSet(ModelViewSet):
queryset = Wbcis.objects.all() # Shouldn't need this anymore
serializer_class = WbcisSerializer
def get_queryset(self):
fruit = self.request.query_params.get("fruit")
....
return get_wbscis(...)
I have an app on appengine that stores some data fields entered by user. I want to prevent redundant entries, i.e. if all fields are same, data should not be entered in database.
(Optional) If identical data is entered, value of a corresponding column "count" should be incremented.
I tried using Django Meta option unique_together for this purpose but it doesn't seem to work. Identical data is still being stored in database. Please help. Here is my code:
class Log(db.Model):
name = db.StringProperty()
location = db.StringProperty()
msg = db.StringProperty()
class Meta:
unique_together = (("name","location","msg"),)
There are some misunderstanding.
Firstly, you are using datastore in your code, not django. There is no unique_together nor Meta options for datastore.
Datastore is a nosql service on appengine.
If you want to make datastore entity to be unique. The easiest way is using key_name.
key_name will be unique. The later entity will replace the old one while they have the same key_name.
For example:
# key_name has length limit (500), hash it can make sure it won't exceed the limit
log = Log(
key_name=str(hash((name,location,msg))),
name=name,
location=location,
msg=msg
)
log.put()
# it will replace any exist log which has the same name, location, msg combination.
# and the item can be Retrieve via key_name directly.
log = Log.get(key_name)
EDIT2:
built-in hash may return different value in different machine. So it is better to use hashlib instead.
you can defined your key_name in many ways, just make sure it won't collision in accident.
for example:
md5: http://docs.python.org/2/library/md5.html
or just append all field together. key_name=name + "|" + location + "|" + msg
for more information:
https://developers.google.com/appengine/docs/python/datastore/entities#Retrieving_an_Entity
If you want to use django on app engine, the model should be defined as:
from django.db import models
class Log(models.Model):
name = models.CharField(max_length=255)
location = models.StringProperty(max_length=255)
msg = models.StringProperty(max_length=255)
class Meta:
unique_together = (("name","location","msg"),)
EDIT3:
Here is a complete example, one for db and the other for ndb.
For ndb, it is quite simple. For db, it is a little bit hard.
from google.appengine.ext import db
from google.appengine.ext import ndb
import webapp2
class Log(db.Model):
name = db.StringProperty()
location = db.StringProperty()
msg = db.StringProperty()
count = db.IntegerProperty()
#classmethod
def key_name(cls, name, location, msg):
return name+"|"+location+"|"+msg
#classmethod
def get(cls, name, location, msg):
return db.get(db.Key.from_path(cls.__name__, cls.key_name(name, location, msg) ))
class nLog(ndb.Model):
name = ndb.StringProperty()
location = ndb.StringProperty()
msg = ndb.StringProperty()
count = ndb.IntegerProperty()
class Test1(webapp2.RequestHandler):
def get(self):
name='test_name'
location = 'test_location'
msg = 'test_msg'
Qkey_name= Log.key_name(name, location, msg)
log = Log(
key_name=Qkey_name,
name=name,
location=location,
msg=msg,
count=0
).put()
if Log.get(name, location, msg) is not None:
Qcount = Log.get(name, location, msg).count
else:
Qcount = 1
class Test2(webapp2.RequestHandler):
def get(self):
name='test_name'
location = 'test_location'
msg = 'test_msg'
Qkey_name = name + "|" + location + "|" + msg
log = nLog(
id=Qkey_name,
name=name,
location=location,
msg=msg,
count=0
).put()
if nLog.get_by_id(Qkey_name) is not None:
Qcount = nLog.get_by_id(Qkey_name).count
else:
Qcount = 1
app = webapp2.WSGIApplication([
(r'/1', Test1),
(r'/2', Test2)
], debug=True)
I am not familiar with django but to solve your problem I would just use some sort of hash of the data and assign it as key_name of the entity. That way you are guaranteed it will be unique, and a counter should be trivial to implement using a put hook.
I have this model in my code:
class Conversation(models.Model):
participants = models.ManyToManyField(User, related_name="message_participants")
and I need to filter this "Conversation" model objects by the "participants" many-to-many field.
meaning: I have for example 3 User objects, so I want to retrieve the only "Conversation" objects that has this 3 Users in it's "participants" field.
I tried doing this:
def get_exist_conv_or_none(sender,recipients):
conv = Conversation.objects.filter(participants=sender)
for rec in recipients:
conv = conv.filter(participants=rec)
where sender is a User object and "recipients" is a list of User objects.
it won't raise error but it gives me the wrong Object of Conversation.
Thanks.
edit:
A more recent try lead me to this:
def get_exist_conv_or_none(sender,recipients):
participants=recipients
participants.append(sender)
conv = Conversation.objects.filter(participants__in=participants)
return conv
which basically have the same problem. It yields Objects which has one or more of the "participants" on the list. but what Im looking for is exact match of the many-to-many object.
Meaning, an Object with the exact "Users" on it's many-to-many relation.
edit 2: My last attempt. still, won't work.
def get_exist_conv_or_none(sender,recipients):
recipients.append(sender)
recipients = list(set(recipients))
conv = Conversation.objects.annotate(count=Count('participants')).filter(participants=recipients[0])
for participant in recipients[1:]:
conv.filter(participants=participant)
conv.filter(count=len(recipients))
return conv
Ok so I found the answer:
In order to make an exact match I have to chain-filter the model and then make sure it has the exact number of arguments it needs to have, so that the many-to-many field will have in it all the objects needed and no more.
I will check for the objects number using annotation: ( https://docs.djangoproject.com/en/dev/topics/db/aggregation/ )
ended up with this code:
def get_exist_conv_or_none(recipients):
conv = Conversation.objects.annotate(count=Count('participants')).filter(participants=recipients[0])
for participant in recipients[1:]:
conv = conv.filter(participants=participant)
conv = conv.filter(count=len(recipients))
return conv
For fast search using database index, I use this code:
class YandexWordstatQueue(models.Model):
regions = models.ManyToManyField(YandexRegion)
regions_cached = models.CharField(max_length=10000, editable=False, db_index=True)
phrase = models.ForeignKey(SearchPhrase, db_index=True)
tstamp = models.DateTimeField(auto_now_add=True)
class YandexWordstatRecord(models.Model):
regions = models.ManyToManyField(YandexRegion)
regions_cached = models.CharField(max_length=10000, editable=False, db_index=True)
phrase = models.ForeignKey(SearchPhrase, db_index=True)
Shows = models.IntegerField()
date = models.DateField(auto_now_add=True)
#receiver(m2m_changed, sender=YandexWordstatRecord.regions.through)
#receiver(m2m_changed, sender=YandexWordstatQueue.regions.through)
def yandexwordstat_regions_changed(sender, **kwargs):
if kwargs.get('action') in ['post_add', 'post_remove']:
instance = kwargs.get('instance')
l = list(instance.regions.values_list('RegionID', flat=True))
l.sort()
instance.regions_cached = json.dumps(l)
instance.save()
This adds overhead when saving, but now I can perform fast filter with this snippet:
region_ids = [1, 2, 3] # or list(some_queryset.values_list(...))
region_ids.sort()
regions_cahed = json.dumps(region_ids)
YandexWordstatQueue.objects.filter(regions_cached=regions_cached)
Still learning Django, so not sure if there's a nice way to do this.
I have a few models with specific attributes (all use Item as base class), and a metadata table (id, language, type, value) used to store any extra attributes that could be potentially associated with instances of any of those models (code below). These models are used with a form / template, simple web-based CRUD.
Right now, I call .save_metadata(...) and .load_metadata(...) explicitly, and use .form_initia(...) to populate the form with metadata that isn't explicitly in the model.
I'm looking for a way to handle this automatically -- basically implementing a model with a variable number of fields, key ones are columns in the model's table, the other ones are rows in the metadata table, and are instance-specific. Is there a way of hooking a method after objects.get(...) or objects.filter(...) etc? I've messed with custom managers and looked into signals, but nothing seems to lead towards an acceptable solution.
class Item(models.Model):
mdata = ['title'] # metadata associated with item
user = models.ForeignKey(User)
created = models.DateTimeField(auto_now_add = True)
status = models.IntegerField(default=0, choices = ([(0,'Staged'), (1,'Published'),(2,'Archived'), ]))
def set_status(self, s):
self.status = s
self.save()
# stores metadata attributes associated with current item
def save_metadata(self, lang, form):
for mt in self.mdata:
try:
md = Metadata.objects.get(item=self, lang=lang, t=mt)
except Metadata.DoesNotExist:
md = Metadata.objects.create(item=self, lang=lang, t=mt)
md.v=form.cleaned_data[mt]
md.save()
# retrieves metadata attributes associated with current item
def load_metadata(self, lang):
for mt in self.mdata:
self.__dict__[mt] = None
try:
self.__dict__[mt] = Metadata.objects.get(item=self, t=mt, lang=lang).v
except Metadata.DoesNotExist:
md = Metadata.objects.filter(item=self, t=mt)
if len(md) > 0:
self.__dict__[mt] = md[0].v
# provides metadata attributes associated with current item needed to populate a form
def form_initial(self, seed=None):
meta = {}
for mt in self.mdata:
meta[mt] = self.__dict__[mt]
#meta[mt] = 'test'
if seed:
meta = dict(meta.items() + seed.items())
return meta
# used to store various metadata associated with models derived from Item
class Metadata(models.Model):
item = models.ForeignKey(Item)
lang = models.CharField(max_length = 8)
t = models.CharField(max_length = 250)
v = models.CharField(max_length = 2500)