Django model Meta option unique_together for appengine - django

I have an app on appengine that stores some data fields entered by user. I want to prevent redundant entries, i.e. if all fields are same, data should not be entered in database.
(Optional) If identical data is entered, value of a corresponding column "count" should be incremented.
I tried using Django Meta option unique_together for this purpose but it doesn't seem to work. Identical data is still being stored in database. Please help. Here is my code:
class Log(db.Model):
name = db.StringProperty()
location = db.StringProperty()
msg = db.StringProperty()
class Meta:
unique_together = (("name","location","msg"),)

There are some misunderstanding.
Firstly, you are using datastore in your code, not django. There is no unique_together nor Meta options for datastore.
Datastore is a nosql service on appengine.
If you want to make datastore entity to be unique. The easiest way is using key_name.
key_name will be unique. The later entity will replace the old one while they have the same key_name.
For example:
# key_name has length limit (500), hash it can make sure it won't exceed the limit
log = Log(
key_name=str(hash((name,location,msg))),
name=name,
location=location,
msg=msg
)
log.put()
# it will replace any exist log which has the same name, location, msg combination.
# and the item can be Retrieve via key_name directly.
log = Log.get(key_name)
EDIT2:
built-in hash may return different value in different machine. So it is better to use hashlib instead.
you can defined your key_name in many ways, just make sure it won't collision in accident.
for example:
md5: http://docs.python.org/2/library/md5.html
or just append all field together. key_name=name + "|" + location + "|" + msg
for more information:
https://developers.google.com/appengine/docs/python/datastore/entities#Retrieving_an_Entity
If you want to use django on app engine, the model should be defined as:
from django.db import models
class Log(models.Model):
name = models.CharField(max_length=255)
location = models.StringProperty(max_length=255)
msg = models.StringProperty(max_length=255)
class Meta:
unique_together = (("name","location","msg"),)
EDIT3:
Here is a complete example, one for db and the other for ndb.
For ndb, it is quite simple. For db, it is a little bit hard.
from google.appengine.ext import db
from google.appengine.ext import ndb
import webapp2
class Log(db.Model):
name = db.StringProperty()
location = db.StringProperty()
msg = db.StringProperty()
count = db.IntegerProperty()
#classmethod
def key_name(cls, name, location, msg):
return name+"|"+location+"|"+msg
#classmethod
def get(cls, name, location, msg):
return db.get(db.Key.from_path(cls.__name__, cls.key_name(name, location, msg) ))
class nLog(ndb.Model):
name = ndb.StringProperty()
location = ndb.StringProperty()
msg = ndb.StringProperty()
count = ndb.IntegerProperty()
class Test1(webapp2.RequestHandler):
def get(self):
name='test_name'
location = 'test_location'
msg = 'test_msg'
Qkey_name= Log.key_name(name, location, msg)
log = Log(
key_name=Qkey_name,
name=name,
location=location,
msg=msg,
count=0
).put()
if Log.get(name, location, msg) is not None:
Qcount = Log.get(name, location, msg).count
else:
Qcount = 1
class Test2(webapp2.RequestHandler):
def get(self):
name='test_name'
location = 'test_location'
msg = 'test_msg'
Qkey_name = name + "|" + location + "|" + msg
log = nLog(
id=Qkey_name,
name=name,
location=location,
msg=msg,
count=0
).put()
if nLog.get_by_id(Qkey_name) is not None:
Qcount = nLog.get_by_id(Qkey_name).count
else:
Qcount = 1
app = webapp2.WSGIApplication([
(r'/1', Test1),
(r'/2', Test2)
], debug=True)

I am not familiar with django but to solve your problem I would just use some sort of hash of the data and assign it as key_name of the entity. That way you are guaranteed it will be unique, and a counter should be trivial to implement using a put hook.

Related

The problem with duplicate nodes in sqlalchemy mptt

On our project, with numerous requests to api in a short time, they are processed in different threads. When a user requests the creation of several instances at once in a table that is inherited from MPTT, this occurs in such a way that the front-end sends several different requests to the back-end in a very short time, each of which calls the code for creating one instance of this table. These requests are processed in different threads. But an unpleasant situation arises, when processing in different threads, instances are assigned the same tree and the same left and right numbers for different table rows. As a result, it turns out that the tree has in the same node several different values ​​for this node.
Please help me to fix this problem. How we can deal with this problem?
models.py:
import sqlalchemy as sa
from sqlalchemy.orm import relationship
from sqlalchemy_mptt import BaseNestedSets
class UserFile(BaseNestedSets):
__tablename__ = "user_files"
id = sa.Column(sa.Integer, primary_key=True)
is_file: bool = sa.Column(sa.Boolean, default=False)
name: str = sa.Column(sa.String(100))
real_name: Optional[str] = sa.Column(sa.String(50))
size: int = sa.Column(sa.Integer, default=0)
downloaded_at: datetime = sa.Column(sa.DateTime, default=datetime.utcnow, nullable=False)
author_id = sa.Column("author_id", sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=True)
author = relationship("User", foreign_keys=[self.author_id])
def __repr__(self) -> str:
return self.name
schemas.py:
#spec_definition
class UserFileSchema(ExtendModelSchema):
class Meta:
model = UserFile
fields = ["id", "name", "is_file", "size", "downloaded_at"]
views.py:
#api.resource("/user/<int:user_id>/files/root/add_file")
class UserFilesAddRootFileResource(SchemaResource):
def post(self, user_id: int) -> dict:
content = request.files.get("file")
item = add_file(user_id, content, filename=content.filename)
return UserFileSchema().dump(item)
user_files.py:
def add_file(user_id: int, content: FileStorage, filename: str) -> UserFile:
size = file_size(content)
final_name = storage.save(None, content, True)
return add_user_file(
user_id=user_id,
name=filename,
is_file=True,
real_name=final_name,
size=size,
)
def add_user_file(
user_id: int,
name: str,
real_name: Optional[str] = None,
size: int = 0,
) -> UserFile:
base_name = name
if is_file:
base_name, ext = storage.splitext(base_name)
result_name = name
item_id = None
item = UserFile(
name=result_name,
real_name=real_name,
is_file=is_file,
size=size,
)
item.save()
return item
I am omitting the definition of some functions from the storage module, which are currently just a layer for working with the os.path.splitext(name) function or writing a file to disk, and they are not relevant to this problem.
I also wanted to ask why some tree nodes can have negative right and left numbers?
We use in our project Flask, SQLAlchemy, sqlalchemy-mptt==0.2.5

Django: how to wrap DB function on top of the field while querying records from that model and inserting using Django Custom Model Field?

I have a text field in the model. Which holds the encrypted string in that field. Also, I have 2 functions in the DB to Encrypt and Decrypt. So, I want to wrap these DB functions on top of that field and save the encrypted string in the DB and while retrieving decrypt function should be wrapped on that field, and the object should have the decrypted string in that attribute.
I am thinking of using the Custom Field in Django. Please suggest to me how to do that?
class User(models.model):
name = models.CharField()
age = models.IntegerField()
email = models.TextField()
for example, let's take the email field, I want to have an encrypted string in that text field.
Let's take DB functions encrypt_email, decrypt_email
when I am running the following
user_objects= User.objects.filter(age__gt = 20)
It should run the query like as follows
SELECT *, decrypt_email(email) as email FROM users where age > 20;
Also when Inserting it should run the query as follows
INSERT INTO users (name, age, email) VALUES ('Rafiu', 25, encrypt_email('abc#gmail.com'));
I have analyzed the Django Framework and implemented it as follows with the custom model field.
from django.db.models.expressions import Col
from django.db.models import Func, TextField, Value
from django.utils.functional import cached_property
class EncryptEmail(Func):
def __init__(self, expression, **extra):
super().__init__(
expression,
**extra
)
function = 'encrypt_email'
template = "%(function)s(%(expressions)s)"
class CustomCol(Col):
def as_sql(self, compiler, connection):
qn = compiler.quote_name_unless_alias
base_sql = "{alias}.{col}".format(alias=qn(self.alias), col=qn(self.target.column))
sql_str = "{f_name}({n_name})".format(f_name='decrypt_email', n_name=base_sql)
return sql_str, []
class EncryptedTextField(TextField):
description = "Values Saved Encrypted and retrieved Decrypted"
def pre_save(self, model_instance, add):
setattr(model_instance, self.attname, EncryptEmail(Value(getattr(model_instance, self.attname))))
return getattr(model_instance, self.attname)
def select_format(self, compiler, sql, params):
sql_str = "{n_name} as {rn_name}".format(n_name=sql, rn_name=self.attname)
return sql_str, params
def get_col(self, alias, output_field=None):
if output_field is None:
output_field = self
if alias != self.model._meta.db_table or output_field != self:
return CustomCol(alias, self, output_field)
else:
return self.cached_col
#cached_property
def cached_col(self):
return CustomCol(self.model._meta.db_table, self)
class User(models.model):
name = models.CharField()
age = models.IntegerField()
email = EncryptedTextField()
It is working as expected.

Show fields based on the value of another field in flask admin

I have database tables with a 'TYPE' column and many other fields. In many cases, certain column values are null based on the value of 'TYPE'.
E.g.
if I have a product table , with TYPE having either 'car' or 'helicopter'. The columns are:
vertical speed, horizontal speed, and horn amplitude.
In the case of 'car' types, vertical speed should always be null , and in the case of 'helicopter' , horn amplitude should always be null.
In flask admin, is there any way to hide the fields from being submitted based on the currently selected TYPE's value?
It is fine if it is a UI level change (i.e. no backend validation is required for security/consistency purposes).
In my real life scenario, there are over 10 columns with 5+ being null in cases, so it would be very helpful if those fields can be removed in the UI (since it makes the form very long and prone to errors).
I am using flask sqlalchemy as the backend for my flask admin.
Fun question. Here is a working solution. So basically you have product types, and each type has certain valid attributes (e.g. car and honking loudness). You can also have general attributes irregardless of the type, e.g. the name of each product.
On the form_prefill, you check what fields are valid for the product type. Then you throw away the invalid fields from the form, and return the form again. It's actually pretty straightforward.
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
import flask_admin as admin
from flask_admin.contrib import sqla
app = Flask(__name__)
app.secret_key = 'arstt'
db = SQLAlchemy(app)
class Product(db.Model):
id = db.Column(db.Integer, primary_key=True)
type = db.Column(db.String)
name = db.Column(db.String)
vertical_speed = db.Column(db.Integer)
rotor_rpm = db.Column(db.Integer)
honking_loudness = db.Column(db.Integer)
def __str__(self):
return "{}".format(self.name)
class ProductView(sqla.ModelView):
general_product_attributes = ['name']
product_attributes_per_product_type_dict = {
'driving': ['honking_loudness'],
'flying': ['rotor_rpm', 'vertical_speed']
}
def on_form_prefill(self, form, id):
product = self.get_one(id)
form_attributes = self.general_product_attributes + self.product_attributes_per_product_type_dict[product.type]
for field in list(form):
if field.name not in form_attributes:
delattr(form, field.name)
return form
db.create_all()
admin = admin.Admin(app, name='Example: SQLAlchemy', template_mode='bootstrap3')
admin.add_view(ProductView(Product, db.session))
helicopter = Product(type='flying', name='helicopter1', vertical_speed=99)
car = Product(type='driving', name='car2', honking_loudness=33)
db.session.add(helicopter)
db.session.add(car)
db.session.commit()
Note that this only works for the edit form, all attribtutes are still being displayed on the create form because it is not certain yet what type an product will be.
You can override create_form for create form and on_prefill_form for edit form.
in this functions you can pass some parameters to fields using form_widget_args
def create_form(self):
form = super(JobsView, self).create_form()
# kw = decide which fields to show or hide
# kw["vertical_speed"]["class"] = "hide"
self.form_widget_args = kw
return form
def on_form_prefill(self, form, id):
# kw = decide which fields to show or hide
# kw["vertical_speed"]["class"] = "hide"
self.form_widget_args = kw

Filtering queryset if one value is greater than another value

I am trying to filter in view my queryset based on relation between 2 fields .
however always getting the error that my field is not defined .
My Model has several calculated columns and I want to get only the records where values of field A are greater than field B.
So this is my model
class Material(models.Model):
version = IntegerVersionField( )
code = models.CharField(max_length=30)
name = models.CharField(max_length=30)
min_quantity = models.DecimalField(max_digits=19, decimal_places=10)
max_quantity = models.DecimalField(max_digits=19, decimal_places=10)
is_active = models.BooleanField(default=True)
def _get_totalinventory(self):
from inventory.models import Inventory
return Inventory.objects.filter(warehouse_Bin__material_UOM__UOM__material=self.id, is_active = true ).aggregate(Sum('quantity'))
totalinventory = property(_get_totalinventory)
def _get_totalpo(self):
from purchase.models import POmaterial
return POmaterial.objects.filter(material=self.id, is_active = true).aggregate(Sum('quantity'))
totalpo = property(_get_totalpo)
def _get_totalso(self):
from sales.models import SOproduct
return SOproduct.objects.filter(product__material=self.id , is_active=true ).aggregate(Sum('quantity'))
totalso = property(_get_totalpo)
#property
def _get_total(self):
return (totalinventory + totalpo - totalso)
total = property(_get_total)
And this is line in my view where I try to get the conditional queryset
po_list = MaterialFilter(request.GET, queryset=Material.objects.filter( total__lte = min_quantity ))
But I am getting the error that min_quantity is not defined
What could be the problem ?
EDIT:
My problem got solved thank you #Moses Koledoye but in the same code I have different issue now
Cannot resolve keyword 'total' into field.Choices are: am, author, author_id, bom, bomversion, code, creation_time, description, id, inventory, is_active, is_production, itemgroup, itemgroup_id, keywords, materialuom, max_quantity, min_quantity, name, pomaterial, produce, product, slug, trigger_quantity, uom, updated_by, updated_by_id, valid_from, valid_to, version, warehousebin
Basically it doesn't show any of my calculated fields I have in my model.
Django cannot write a query which is conditioned on a field whose value is unknown. You need to use a F expression for this:
from django.db.models import F
queryset = Material.objects.filter(total__lte = F('min_quantity'))
And your FilterSet becomes:
po_list = MaterialFilter(request.GET, queryset = Material.objects.filter(total__lte=F('min_quantity')))
From the docs:
An F() object represents the value of a model field or annotated
column. It makes it possible to refer to model field values and
perform database operations using them without actually having to pull
them out of the database into Python memory

Django - processing objects after .filter(...) or .all(...)

Still learning Django, so not sure if there's a nice way to do this.
I have a few models with specific attributes (all use Item as base class), and a metadata table (id, language, type, value) used to store any extra attributes that could be potentially associated with instances of any of those models (code below). These models are used with a form / template, simple web-based CRUD.
Right now, I call .save_metadata(...) and .load_metadata(...) explicitly, and use .form_initia(...) to populate the form with metadata that isn't explicitly in the model.
I'm looking for a way to handle this automatically -- basically implementing a model with a variable number of fields, key ones are columns in the model's table, the other ones are rows in the metadata table, and are instance-specific. Is there a way of hooking a method after objects.get(...) or objects.filter(...) etc? I've messed with custom managers and looked into signals, but nothing seems to lead towards an acceptable solution.
class Item(models.Model):
mdata = ['title'] # metadata associated with item
user = models.ForeignKey(User)
created = models.DateTimeField(auto_now_add = True)
status = models.IntegerField(default=0, choices = ([(0,'Staged'), (1,'Published'),(2,'Archived'), ]))
def set_status(self, s):
self.status = s
self.save()
# stores metadata attributes associated with current item
def save_metadata(self, lang, form):
for mt in self.mdata:
try:
md = Metadata.objects.get(item=self, lang=lang, t=mt)
except Metadata.DoesNotExist:
md = Metadata.objects.create(item=self, lang=lang, t=mt)
md.v=form.cleaned_data[mt]
md.save()
# retrieves metadata attributes associated with current item
def load_metadata(self, lang):
for mt in self.mdata:
self.__dict__[mt] = None
try:
self.__dict__[mt] = Metadata.objects.get(item=self, t=mt, lang=lang).v
except Metadata.DoesNotExist:
md = Metadata.objects.filter(item=self, t=mt)
if len(md) > 0:
self.__dict__[mt] = md[0].v
# provides metadata attributes associated with current item needed to populate a form
def form_initial(self, seed=None):
meta = {}
for mt in self.mdata:
meta[mt] = self.__dict__[mt]
#meta[mt] = 'test'
if seed:
meta = dict(meta.items() + seed.items())
return meta
# used to store various metadata associated with models derived from Item
class Metadata(models.Model):
item = models.ForeignKey(Item)
lang = models.CharField(max_length = 8)
t = models.CharField(max_length = 250)
v = models.CharField(max_length = 2500)