MySQLdb commit() method not saving data - python-2.7

I've got a class to perform all the SQL tasks in my code. The problem is when I call a method inside this class to update the database, the self.dbConn.commit() line is not saving the data.
What am I missing? I'm sure it's a newbie error but I can't find the problem.
If more code examples are needed I can provide them.
Cheers!
This is the class itself:
class dbActivities:
def __init__(self):
self.dbConn = my.connect("xxx","xxx","xxx","xxx")
self.dbCursor = self.dbConn.cursor()
def updateDB(self, sql):
try:
self.dbCursor.execute(sql)
self.dbConn.commit()
return True
except:
return False
And that's how I'm calling the method:
dbHandler.updateDB("UPDATE xxx SET token = {}, WHERE xxx = {}".format(xxx, xxx))

Related

Django Rest Framework and Channels, You cannot call this from an async context

What i am trying to do is to nest a DRF model serializer into another model serialiser's field like so
class username_serial(ModelSerializer):
class Meta:
model = User
fields = ['username','email']
class game_serial(ModelSerializer):
user_01 = username_serial()
class Meta:
model = game
fields = ['id','user_01','user_02','is_private','is_accepted']
Error :
Exception inside application: You cannot call this from an async
context - use a thread or sync_to_async. Traceback (most recent call
last): File
"C:\Users\baza\Desktop\production\venv\lib\site-packages\django\db\models\fields\related_descriptors.py",
line 173, in get
rel_obj = self.field.get_cached_value(instance) File "C:\Users\baza\Desktop\production\venv\lib\site-packages\django\db\models\fields\mixins.py",
line 15, in get_cached_value
return instance._state.fields_cache[cache_name] KeyError: 'user_01'
This works normally without Django Chennels because channels is async and i can't use sync code with, works fine by using:
os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
In the settings file but it's not a safe approach when it comes to production.
i tried using channel's database_sync_to_async as a decorator and as well as a function with a SerializerMethodField like so:
class game_serial(ModelSerializer):
user_01 = SerializerMethodField(read_only=True)
#database_sync_to_async
def get_user_01(self, obj):
username = obj.user_01.username
return str(username)
class Meta:
model = game
fields = ['id','user_01','user_02','is_private','is_accepted']
but i get back:
[OrderedDict([('id', 23), ('user_01', <coroutine object SyncToAsync.__call__ at 0x0000000005DF6678>), ('user_02', None), ('is_private', False), ('is_accepted', False)]), OrderedDict([('id', 24), ('user_01', <coroutine object SyncToAsync.__call__ at 0
x0000000005DF6D58>), ('user_02', None), ('is_private', False), ('is_accepted', False)])]
with an
Exception inside application: Object of type 'coroutine' is not JSON
serializable
so i guess that JSON couldn't serialize those "coroutine" objects and normally i should or "want" to get the actual values of them instead.
How can i do the task? any workaround or other methods are welcomed, thank you in advance.
in consumers.py ..
games = await database_sync_to_async(self.get_games)()
serialized_games = game_serial(games, many=True)
await self.send({
"type": "websocket.send",
'text': json.dumps(serialized_games.data)
})
def get_games(self):
return list(game.objects.all())
I never used Django Channels but I know Django and async.
I'll be honest, I don't like these hacky decorators. And I don't think running such a simple task in a thread is a good idea.
You have an obj, so you were able to query the DB earlier. If so, there's a place without async context or async context where accessing the DB works. In the error message user_01 is not found in the "cached" object from the DB. So just prefetch what you need before.
def get_queryset(self):
return game_serial.objects.select_related('user_01')
class game_serial(ModelSerializer):
user_01 = serializers.CharField(source='user_01.username')
This way you don't have problems with this sync-to-async magic, it's more efficient and easier to reason about.
EDIT:
I repeat, you should select related where you fetch the data. After you added another example, I can suggest something like that
def get_games(self):
return list(game.objects.select_related('user_01').all())
and it will work just fine.
You can also try
#database_sync_to_async
def get_games(self):
return list(game.objects.select_related('user_01').all())
and
serialized_games = await game_serial(games, many=True)
In both cases this serializer will work just fine.
class game_serial(ModelSerializer):
user_01 = serializers.CharField(source='user_01.username')
class Meta:
model = game
fields = ['id','user_01','user_02','is_private','is_accepted']

RecursionError: when using factory boy

I can't use factory boy correctly.
That is my factories:
import factory
from harrispierce.models import Article, Journal, Section
class JournalFactory(factory.Factory):
class Meta:
model = Journal
name = factory.sequence(lambda n: 'Journal%d'%n)
#factory.post_generation
def sections(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
if extracted:
# A list of groups were passed in, use them
for section in extracted:
self.sections.add(section)
class SectionFactory(factory.Factory):
class Meta:
model = Section
name = factory.sequence(lambda n: 'Section%d'%n)
and my test:
import pytest
from django.test import TestCase, client
from harrispierce.factories import JournalFactory, SectionFactory
#pytest.mark.django_db
class TestIndex(TestCase):
#classmethod
def setUpTestData(cls):
cls.myclient = client.Client()
def test_index_view(self):
response = self.myclient.get('/')
assert response.status_code == 200
def test_index_content(self):
section0 = SectionFactory()
section1 = SectionFactory()
section2 = SectionFactory()
print('wijhdjk: ', section0)
journal1 = JournalFactory.create(sections=(section0, section1, section2))
response = self.myclient.get('/')
print('wijhdjk: ', journal1)
self.assertEquals(journal1.name, 'Section0')
self.assertContains(response, journal1.name)
But I get this when running pytest:
journal1 = JournalFactory.create(sections=(section0, section1, section2))
harrispierce_tests/test_index.py:22:
RecursionError: maximum recursion depth exceeded while calling a Python object
!!! Recursion detected (same locals & position)
One possible issue would be that you're not using the proper Factory base class: for a Django model, use factory.django.DjangoModelFactory.
This shouldn't cause the issue you have, though; a full stack trace would be useful.
Try to remove the #factory.post_generation section, and see whether you get a proper Journal object; then inspect what parameters where passed.
If this is not enough to fix your code, I suggest opening an issue on the factory_boy repository, with a reproducible test case (there are already some branches/commits attempting to reproduce a reported bug, which can be used as a template).

Convert POST to PUT with Tastypie

Full Disclosure: Cross posted to Tastypie Google Group
I have a situation where I have limited control over what is being sent to my api. Essentially there are two webservices that I need to be able to accept POST data from. Both use plain POST actions with urlencoded data (basic form submission essentially).
Thinking about it in "curl" terms it's like:
curl --data "id=1&foo=2" http://path/to/api
My problem is that I can't update records using POST. So I need to adjust the model resource (I believe) such that if an ID is specified, the POST acts as a PUT instead of a POST.
api.py
class urlencodeSerializer(Serializer):
formats = ['json', 'jsonp', 'xml', 'yaml', 'html', 'plist', 'urlencoded']
content_types = {
'json': 'application/json',
'jsonp': 'text/javascript',
'xml': 'application/xml',
'yaml': 'text/yaml',
'html': 'text/html',
'plist': 'application/x-plist',
'urlencoded': 'application/x-www-form-urlencoded',
}
# cheating
def to_urlencoded(self,content):
pass
# this comes from an old patch on github, it was never implemented
def from_urlencoded(self, data,options=None):
""" handles basic formencoded url posts """
qs = dict((k, v if len(v)>1 else v[0] )
for k, v in urlparse.parse_qs(data).iteritems())
return qs
class FooResource(ModelResource):
class Meta:
queryset = Foo.objects.all() # "id" = models.AutoField(primary_key=True)
resource_name = 'foo'
authorization = Authorization() # only temporary, I know.
serializer = urlencodeSerializer()
urls.py
foo_resource = FooResource
...
url(r'^api/',include(foo_resource.urls)),
)
In #tastypie on Freenode, Ghost[], suggested that I overwrite post_list() by creating a function in the model resource like so, however, I have not been successful in using this as yet.
def post_list(self, request, **kwargs):
if request.POST.get('id'):
return self.put_detail(request,**kwargs)
else:
return super(YourResource, self).post_list(request,**kwargs)
Unfortunately this method isn't working for me. I'm hoping the larger community could provide some guidance or a solution for this problem.
Note: I cannot overwrite the headers that come from the client (as per: http://django-tastypie.readthedocs.org/en/latest/resources.html#using-put-delete-patch-in-unsupported-places)
I had a similar problem on user creation where I wasn't able to check if the record already existed. I ended up creating a custom validation method which validated if the user didn't exist in which case post would work fine. If the user did exist I updated the record from the validation method. The api still returns a 400 response but the record is updated. It feels a bit hacky but...
from tastypie.validation import Validation
class MyValidation(Validation):
def is_valid(self, bundle, request=None):
errors = {}
#if this dict is empty validation passes.
my_foo = foo.objects.filter(id=1)
if not len(my_foo) == 0: #if object exists
foo[0].foo = 'bar' #so existing object updated
errors['status'] = 'object updated' #this will be returned in the api response
return errors
#so errors is empty if object does not exist and validation passes. Otherwise object
#updated and response notifies you of this
class FooResource(ModelResource):
class Meta:
queryset = Foo.objects.all() # "id" = models.AutoField(primary_key=True)
validation = MyValidation()
With Cathal's recommendation I was able to utilize a validation function to update the records I needed. While this does not return a valid code... it works.
from tastypie.validation import Validation
import string # wrapping in int() doesn't work
class Validator(Validation):
def __init__(self,**kwargs):
pass
def is_valid(self,bundle,request=None):
if string.atoi(bundle.data['id']) in Foo.objects.values_list('id',flat=True):
# ... update code here
else:
return {}
Make sure you specify the validation = Validator() in the ModelResource meta.

SQLAlchemy query not retrieving committed data in alternate session

I have a problem where I insert a database item using a SQLAlchemy / Tastypie REST interface, but the item is missing when subsequently get the list of items. It shows up only after I get the list of items a second time.
I am using SQLAlchemy with Tastypie/Django running on Apache via mod_wsgi. I use a singleton Database Manager class to hold my engine and declarative_base, and with Tastypie, a separate class to get the session and make sure I roll-back if there is a problem with the commit. As in the update below, the problem occurs when I don't close my session after inserting. Why is this necessary?
My original code was like this:
Session = scoped_session(sessionmaker(autoflush=True))
# Singleton Database Manager class for managing session
class DatabaseManager():
engine = None
base = None
def ready(self):
host='mysql+mysqldb://etc...'
if self.engine and self.base:
return True
else:
try:
self.engine = create_engine(host, pool_recycle=3600)
self.base = declarative_base(bind=self.engine)
return True
except:
return False
def getSession(self):
if self.ready():
session = Session()
session.configure(bind=self.engine)
return session
else:
return None
DM = DatabaseManager()
# A session class I use with Tastypie to ensure the session is destroyed at the
# end of the transaction, because Tastypie creates singleton Resources used for
# all threads
class MySession:
def __init__(self):
self.s = DM.getSession()
def safeCommit(self):
try:
self.s.commit()
except:
self.s.rollback()
raise
def __del__(self):
try:
self.s.commit()
except:
self.s.rollback()
raise
# ... Then ... when I get requests through Apache/mod_wsgi/Django/Tastypie
# First Request
obj_create():
db = MySession()
print db.s.query(DBClass).count() # returns 4
newItem = DBClass()
db.s.add(newItem)
db.s.safeCommit()
print db.s.query(DBClass).count() # returns 5
# Second Request after First Request returns
obj_get_list():
db = MySession()
print db.s.query(DBClass).count() # returns 4 ... should be 5
# Third Request is okay
obj_get_list():
db = MySession()
print db.s.query(DBClass).count() # returns 5
UPDATE
After further digging, it appears that the problem is my session needed to be closed after creating. Perhaps because Tastypie's object_create() adds the SQLAlchemy object to it's bundle, and I don't know what happens after it leaves the function's scope:
obj_create():
db = MySession()
newItem = DBClass()
db.s.add(newItem)
db.s.safeCommit()
copiedObj = copyObj(newItem) # copy SQLAlchemy record into non-sa object (see below)
db.s.close()
return copiedObj
If someone cares to explain this in an answer, I can close the question. Also, for those who are curious, I copy my object out of SQLAlchemy like this:
class Struct:
def __init__(self, **entries):
self.__dict__.update(entries)
class MyTastypieResource(Resource):
...
def copyObject(self, object):
base = {}
# self._meta is part of my tastypie resource
for p in class_mapper(self._meta.object_class).iterate_properties:
if p.key not in base and p.key not in self._meta.excludes:
base[p.key] = getattr(object,p.key)
return Struct(**base)
The problem was resolved by closing my session. The update in the answer didn't solve the problem fully - I ended up adding a middleware class to close the session at the end of a transaction. This ensured everything was written to the database. The middleware looks a bit like this:
class SQLAlchemySessionMiddleWare(object):
def process_response(self, request, response):
try:
session = MyDatabaseManger.getSession()
session.commit()
session.close()
except Exception, err:
pass
return response
def process_exception(self, request, exception):
try:
session = MyDatabaseManger.getSession()
session.rollback()
session.close()
except Exception, err:
pass

Django: Overriding the save() method: how do I call the delete() method of a child class

The setup =
I have this class, Transcript:
class Transcript(models.Model):
body = models.TextField('Body')
doPagination = models.BooleanField('Paginate')
numPages = models.PositiveIntegerField('Number of Pages')
and this class, TranscriptPages(models.Model):
class TranscriptPages(models.Model):
transcript = models.ForeignKey(Transcript)
order = models.PositiveIntegerField('Order')
content = models.TextField('Page Content', null=True, blank=True)
The Admin behavior I’m trying to create is to let a user populate Transcript.body with the entire contents of a long document and, if they set Transcript.doPagination = True and save the Transcript admin, I will automatically split the body into n Transcript pages.
In the admin, TranscriptPages is a StackedInline of the Transcript Admin.
To do this I’m overridding Transcript’s save method:
def save(self):
if self.doPagination:
#do stuff
super(Transcript, self).save()
else:
super(Transcript, self).save()
The problem =
When Transcript.doPagination is True, I want to manually delete all of the TranscriptPages that reference this Transcript so I can then create them again from scratch.
So, I thought this would work:
#do stuff
TranscriptPages.objects.filter(transcript__id=self.id).delete()
super(Transcript, self).save()
but when I try I get this error:
Exception Type: ValidationError
Exception Value: [u'Select a valid
choice. That choice is not one of the
available choices.']
... and this is the last thing in the stack trace before the exception is raised:
.../django/forms/models.py in save_existing_objects
pk_value = form.fields[pk_name].clean(raw_pk_value)
Other attempts to fix:
t =
self.transcriptpages_set.all().delete()
(where self = Transcript from the
save() method)
looping over t (above) and deleting each item individually
making a post_save signal on TranscriptPages that calls the delete method
Any ideas? How does the Admin do it?
UPDATE: Every once in a while as I'm playing around with the code I can get a different error (below), but then it just goes away and I can't replicate it again... until the next random time.
Exception Type:
MultiValueDictKeyError Exception
Value: "Key 'transcriptpages_set-0-id'
not found in "
Exception Location:
.../django/utils/datastructures.py in
getitem, line 203
and the last lines from the trace:
.../django/forms/models.py in _construct_form
form = super(BaseInlineFormSet, self)._construct_form(i, **kwargs)
.../django/utils/datastructures.py in getitem
pk = self.data[pk_key]
In the end it was a matter of timing. When deleting only a single child object out of many, there was no problem. If I was deleting too many child objects at once, the error could happen, because the delete action was attempting to reference ids that were not around. This is why nothing worked, not signals, not [object]_set. I fixed it by using jquery to set a hidden variable in the edit form for the child object, which caused the object to first process the update (slowing down it's processing) and THEN the delete.
You are probably trying to access the Transaction.id before is has been created. Additionally, you can try to access the TransactionPage objects through the Transaction object via transactionpage_set (see query docs about FOO_set notation).
def save(self):
super(Transcript, self).save()
if self.doPagination:
self.transaction_set.all().delete() # .all() may be optional
pages = ... # whatever you do to split the content
self.numPages = len(pages)
self.save() # yes, a second save if you store numPages in Transaction
for page_number in range(self.numPages):
TransactionPage.objects.create(content=pages[page_number],
order=page_number, transaction=self)
You could also switch to not storing numPages in the Transaction and access it via a property instead.
class Transaction(models.Model):
# ...
# replace numPages with
#property
def page_count(self):
return self.transactionpage_set.count() or 1
And then if you took it one step further you could always use the TransactionPage objects for display purposes. This would allow you to get rid of the extra self.save() call in the above save() method. It will also let you simplify your templates by always displaying TransactionPage.content instead of conditionally displaying Transaction.body if you are paginating and TransactionPage.content otherwise.
class Transaction(models.Model):
body = models.TextField()
paginate = models.BooleanField()
#property
def page_count(self):
return self.transactionpage_set.count() # no more "or 1" cruft!
def save(self):
super(Transcript, self).save()
self.transaction_set.delete() # might need an .all() before .delete()
if self.paginate:
# Do whatever you do to split the body into pages.
pages = ...
else:
# The only page is the entire body.
pages = [self.body]
for (page_number, page_content) in enumerate(pages):
TransactionPage.objects.create(content=page_content,
order=page_number, transaction=self)
Another possible solution might be to override save_formset() in the ModelAdmin to prevent the update completely:
def save_formset(self, request, form, formset, change):
if (form.cleaned_data['do_pagination'] and
formset.model == TranscriptPages):
formset.changed_objects = []
formset.new_objects = []
formset.deleted_objects = []
else:
formset.save()
Then, you can do whatever you like in Modal.save(). Note that there's probably a more elegant/breakproof way to stop the formset from processing if one wanted to dig into the internals a bit more.