EDIT 2022-10-04 18:40
I've tried using bulk_update and bulk_create as these method only query database once but still have the same issue
would appreciate any help/explanation on this issue
'''
Task to edit data correction forms (DCF) online
'''
#shared_task(bind=True)
def DCF_edition(self):
print(timezone.now())
DCF_BEFORE_UPDATE = pd.DataFrame.from_records(DataCorrectionForm.objects.all().values())
if not DCF_BEFORE_UPDATE.empty :
DCF_BEFORE_UPDATE = DCF_BEFORE_UPDATE.rename(columns={"patient": "pat"})
DCF_BEFORE_UPDATE = DCF_BEFORE_UPDATE.astype({'record_date': str,'created_date': str})
DCF_BEFORE_UPDATE['dcf_status'] = DCF_BEFORE_UPDATE.apply(lambda status: 0, axis=1)
# list of dataframe to concat
data = []
# load queries definition
queries = queries_definitions()
# print(queries)
if not queries.empty:
for index, row in queries.iterrows():
print('Query ide',row['ide'])
# print(row['ide'],row['query_type'],row['crf_name'].lower(),row['crf identification date'],row['variable_name'],row['variable_label'],row['query_condition'],row['fields_to_display'])
try:
missing_or_inconsistent = missing_or_inconsistent_data(row['ide'],row['query_type'],row['crf_name'].lower(),row['crf identification date'],row['variable_name'],row['variable_label'],row['query_condition'],row['fields_to_display']) #.iloc[:10] #to limit rows
missing_or_inconsistent.columns.values[2] = 'record_date' # rename the date column (that have database name)
missing_or_inconsistent['dcf_ide'] = str(row['ide']) + '_' + row['variable_name'] + '_' + missing_or_inconsistent[row['crf primary key']].astype(str)
missing_or_inconsistent['category'] = row['query_type']
missing_or_inconsistent['crf'] = row['crf_name']
missing_or_inconsistent['crf_ide'] = missing_or_inconsistent[row['crf primary key']]
missing_or_inconsistent['field_name'] = row['variable_name']
missing_or_inconsistent['field_label'] = row['variable_label']
missing_or_inconsistent['field_value'] = missing_or_inconsistent[row['variable_name']]
missing_or_inconsistent['message'] = row['query_message']
missing_or_inconsistent['query_id'] = 'Query ide ' + str(row['ide'])
missing_or_inconsistent['dcf_status'] = 1
missing_or_inconsistent['created_date'] = timezone.now()
missing_or_inconsistent['deactivated'] = False
missing_or_inconsistent['comments'] = None
data.append(missing_or_inconsistent[['ide','dcf_ide','category','crf','crf_ide','pat','record_date','field_name','field_label','message','field_value','dcf_status','created_date','query_id','deactivated','comments']])
dcf = pd.concat(data)
except Exception as e:
Log.objects.create(dcf_edition_status=0,dcf_edition_exception=str(e)[:200])
continue
DCF_AFTER_UPDATE = pd.concat([DCF_BEFORE_UPDATE,dcf])
DCF_AFTER_UPDATE['duplicate'] = DCF_AFTER_UPDATE.duplicated(subset=['dcf_ide'],keep='last')
DCF_AFTER_UPDATE['dcf_status'] = DCF_AFTER_UPDATE.apply(lambda row: 2 if row['duplicate'] else row['dcf_status'],axis=1)
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.drop_duplicates(subset=['dcf_ide'],keep='first').drop(columns=['duplicate'])
DCF_AFTER_UPDATE.rename(columns = {'pat':'patient',}, inplace = True)
# Cast date into string format to be able to dumps data
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.astype({'record_date': str}) if not DCF_AFTER_UPDATE.empty else DCF_AFTER_UPDATE
records_to_update = [
DataCorrectionForm(
ide=record['ide'],
dcf_ide=record['dcf_ide'],
category=record['category'],
crf=record['crf'],
crf_ide=record['crf_ide'],
patient=record['patient'],
record_date=record['record_date'],
field_name=record['field_name'],
field_label=record['field_label'],
message=record['message'],
field_value=record['field_value'],
dcf_status=record['dcf_status'],
created_date=record['created_date'],
query_id=record['query_id'],
deactivated=record['deactivated'],
comments=record['comments']
) for i, record in DCF_AFTER_UPDATE[(DCF_AFTER_UPDATE['dcf_status'] != 1)].iterrows()
]
if records_to_update:
DataCorrectionForm.objects.bulk_update(records_to_update,['dcf_status'])
records_to_create = [
DataCorrectionForm(
dcf_ide=record['dcf_ide'],
category=record['category'],
crf=record['crf'],
crf_ide=record['crf_ide'],
patient=record['patient'],
record_date=record['record_date'],
field_name=record['field_name'],
field_label=record['field_label'],
message=record['message'],
field_value=record['field_value'],
dcf_status=record['dcf_status'],
created_date=record['created_date'],
query_id=record['query_id'],
deactivated=record['deactivated'],
comments=record['comments']
) for i, record in DCF_AFTER_UPDATE[(DCF_AFTER_UPDATE['dcf_status'] == 1)].iterrows()
]
if records_to_create:
DataCorrectionForm.objects.bulk_create(records_to_create)
EDIT 2022-10-04 13:40
I've tried to "optimized" code using update_or_create() method but doesn't change anything
I still have an OperationalError with the line DataCorrectionForm.objects.update_or_create(...)
How can I update my database?
'''
Task to edit data correction forms (DCF) online
'''
#shared_task(bind=True)
def DCF_edition(self):
DCF_BEFORE_UPDATE = pd.DataFrame.from_records(DataCorrectionForm.objects.all().values())
if not DCF_BEFORE_UPDATE.empty :
DCF_BEFORE_UPDATE.drop(columns=['ide'])
DCF_BEFORE_UPDATE = DCF_BEFORE_UPDATE.rename(columns={"patient": "pat"})
DCF_BEFORE_UPDATE = DCF_BEFORE_UPDATE.astype({'record_date': str,'created_date': str})
DCF_BEFORE_UPDATE['dcf_status'] = DCF_BEFORE_UPDATE.apply(lambda status: 0, axis=1)
# list of dataframe to concat
data = []
# load queries definition
queries = queries_definitions()
if not queries.empty:
for index, row in queries.iterrows()
try:
missing_or_inconsistent = missing_or_inconsistent_data(row['ide'],row['query_type'],row['crf_name'].lower(),row['crf identification date'],row['variable_name'],row['variable_label'],row['query_condition'],row['fields_to_display']) #.iloc[:10] #to limit rows
missing_or_inconsistent.columns.values[2] = 'record_date' # rename the date column (that have database name)
missing_or_inconsistent['dcf_ide'] = str(row['ide']) + '_' + row['variable_name'] + '_' + missing_or_inconsistent[row['crf primary key']].astype(str)
missing_or_inconsistent['category'] = row['query_type']
missing_or_inconsistent['crf'] = row['crf_name']
missing_or_inconsistent['crf_ide'] = missing_or_inconsistent[row['crf primary key']]
missing_or_inconsistent['field_name'] = row['variable_name']
missing_or_inconsistent['field_label'] = row['variable_label']
missing_or_inconsistent['field_value'] = missing_or_inconsistent[row['variable_name']]
missing_or_inconsistent['message'] = row['query_message']
missing_or_inconsistent['DEF'] = 'Query ide ' + str(row['ide'])
missing_or_inconsistent['dcf_status'] = 1
missing_or_inconsistent['created_date'] = timezone.now()
missing_or_inconsistent['deactivated'] = False
missing_or_inconsistent['comments'] = None
data.append(missing_or_inconsistent[['dcf_ide','category','crf','crf_ide','pat','record_date','field_name','field_label','message','field_value','dcf_status','created_date','DEF','deactivated','comments']])
dcf = pd.concat(data)
except Exception as e:
Log.objects.create(dcf_edition_status=0,dcf_edition_exception=str(e)[:200])
continue
DCF_AFTER_UPDATE = pd.concat([DCF_BEFORE_UPDATE,dcf])
DCF_AFTER_UPDATE['duplicate'] = DCF_AFTER_UPDATE.duplicated(subset=['dcf_ide'],keep='last')
DCF_AFTER_UPDATE['dcf_status'] = DCF_AFTER_UPDATE.apply(lambda row: 2 if row['duplicate'] else row['dcf_status'],axis=1)
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.drop_duplicates(subset=['dcf_ide'],keep='first').drop(columns=['duplicate'])
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.drop(['DEF'], axis=1)
DCF_AFTER_UPDATE.rename(columns = {'pat':'patient',}, inplace = True)
# Cast date into string format to be able to dumps data
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.astype({'record_date': str}) if not DCF_AFTER_UPDATE.empty else DCF_AFTER_UPDATE
records = DCF_AFTER_UPDATE.to_dict(orient='records')
for record in records:
DataCorrectionForm.objects.update_or_create(
dcf_ide=record['dcf_ide'], # filter to search for existing objects => should not be pass to default (if not IntegrityError)
defaults = {
'category':record['category'],
'crf':record['crf'],
'crf_ide':record['crf_ide'],
'patient':record['patient'],
'record_date':record['record_date'],
'field_name':record['field_name'],
'field_label':record['field_label'],
'message':record['message'],
'field_value':record['field_value'],
'dcf_status':record['dcf_status'],
'created_date':record['created_date'],
# 'DEF':record['DEF'],
'deactivated':record['deactivated'],
'comments':record['comments']
}
)
Log.objects.create(dcf_edition_status=1)
return True
EDIT 2022-10-03 17:00
in fact reading CAVEATS:
The development server creates a new thread for each request it
handles, negating the effect of persistent connections. Don’t enable
them during development.
EDIT 2022-10-03 16:00
Django 2.2.5
I have tried to set DATABASES parameter CONN_MAX_AGE as per Django documentation but it doesn't change anythings
Default: 0
The lifetime of a database connection, as an integer of seconds. Use 0
to close database connections at the end of each request — Django’s
historical behavior — and None for unlimited persistent connections.
I use Celery task and got an error I do not understand.
I loop over a table (that contain query definitions) to edit missing/inconsistent data in a database (using API) and registered discrepencies in another table.
If I run query one at a time, it works but when I try to loop over queries, I got an error
OperationalError('server closed the connection unexpectedly\n\tThis probably means the server terminated abnormally\n\tbefore or while processing the request.\n')
def DCF_edition(self):
DCF_BEFORE_UPDATE = pd.DataFrame.from_records(DataCorrectionForm.objects.all().values())
DCF_BEFORE_UPDATE = DCF_BEFORE_UPDATE.astype({'record_date': str,'created_date': str}) if not DCF_BEFORE_UPDATE.empty else DCF_BEFORE_UPDATE
data = []
# load queries definition
queries = queries_definitions()
if not queries.empty:
for index, row in queries.iterrows():
try:
missing_or_inconsistent = missing_or_inconsistent_data(row['ide'],row['query_type'],row['crf_name'].lower(),row['crf identification
data.append(missing_or_inconsistent[['dcf_ide','category','crf','crf_ide','pat','record_date','field_name','field_label','message','field_value','dcf_status','DEF','deactivated']])
DCF_AFTER_UPDATE = pd.concat(data)
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.drop_duplicates(keep='last')
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.drop(['DEF'], axis=1)
DCF_AFTER_UPDATE.rename(columns = {'pat':'patient',}, inplace = True)
except Exception as e:
Log.objects.create(dcf_edition_status=0,dcf_edition_exception=str(e)[:200])
continue
# Cast date into string format to be able to dumps data
DCF_AFTER_UPDATE = DCF_AFTER_UPDATE.astype({'record_date': str}) if not DCF_AFTER_UPDATE.empty else DCF_AFTER_UPDATE
records = json.loads(json.dumps(list(DCF_AFTER_UPDATE.T.to_dict().values())))
for record in records:
if not DCF_BEFORE_UPDATE.empty and record['dcf_ide'] in DCF_BEFORE_UPDATE.values:
DataCorrectionForm.objects.filter(dcf_ide=record['dcf_ide']).update(dcf_status=2)
else:
DataCorrectionForm.objects.get_or_create(**record)
# resolved dcf => status=0
if not DCF_BEFORE_UPDATE.empty:
records = json.loads(json.dumps(list(DCF_BEFORE_UPDATE.T.to_dict().values())))
for record in records:
if record['dcf_ide'] not in DCF_AFTER_UPDATE.values:
DataCorrectionForm.objects.filter(dcf_ide=record['dcf_ide']).update(dcf_status=0)
Log.objects.create(dcf_edition_status=1)
return True
The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior — and None for unlimited persistent connections.
It seems that your task is long running task and need to hold the db connection for a long period. Did you try to set it to None
DATABASES = {
'default': env.db(),
}
# https://docs.djangoproject.com/en/3.1/ref/settings/#conn-max-age
DATABASES['default']['CONN_MAX_AGE'] = None
How long does your task need to finish? It could be another problem with server database setting, ex tcp_keepalives_ilde..
I did the code for insert records from so_parts table to so_bo table using Query...How can I use ORM method to do this kind of job. Is there any other way(best)to do that? Here is my code`
`
#api.multi
def save_rapair_parts(self, vals):
#get todays date and convert it to string
created_date = datetime.datetime.today().strftime("%m/%d/%Y")
str_date = str(created_date)
so_p_id = self.so_p_id.id
bo_status = self.bo_status
so_part_t = self.so_part_t
so_part_sno = self.so_part_sno
product = self.so_part_product
product_str = 'Repair '+str(product)
part_id = self.id
bench_order_table.search(['id','bo_sno','created_date','bo_number','rep_description','bo_status'])
#insert details intoso bench orders
`
if so_part_t=='r_b':
try:
sequence = self.env['ir.sequence'].next_by_code('so.benchorder') or '/'
str_sequence = str(sequence)
query = """SELECT so_work_authorization FROM my_depots_so WHERE id=%d """ % (so_p_id)
self.env.cr.execute(query)
result = self.env.cr.fetchall()
result_number = json.dumps(result, ensure_ascii=False)
strip_number = result_number.strip('\' \" [] ')
work_auth_no = str(strip_number)
work_auth_no += "-"
work_auth_no += str_sequence
insert ="""INSERT INTO my_depots_so_bo(id,so_bo_id,bo_sno,created_date,bo_number,rep_description,bo_status) values %s """
parameters = (part_id,so_p_id,so_part_sno,str_date,work_auth_no,product_str,bo_status)
self.env.cr.execute(insert,(parameters,))
my_depots_bo(id,bo_sno,created_date,bo_number,rep_description,bo_status) values %s """
# self.env.cr.execute(insert_query, (parameters,))
except Exception:
print "Error in inserting values"`
yes there is a better way because when you use ORM
method you also checks access right for user to:
for your select query:
rec = self.env['my.depots.so'].search_read(['id', '=', so_p_id], ['so_work_authorization'])
if rec:
rec = rec[0] # search_read return a list of dictionary
so_work_authorization = rec['so_work_authorization']
# and do what ever you want with the result
# to create
# call create method witch accept a dictionary
# field_name : value
new_rec = self.env['my.depots.so.bo'].create({
'so_bo_id': so_p_id, # many2one must be an integer value
'bo_sno': bo_nso_value,
'bo_number': value_of_number,
# ....
# ....
# add al field
}) # create return the new created record as model object
for inserting use: self.env['model.name'].create(vals)
for updating use : self.env['model.name'].write(vals)
using ORM method makes sure that user don't pass the security access rigths
Hope you get the idea
I want to create a funtion which takes 3 arguments which are as follows.
path = "['pma']['device']['clock']"
key = 'key'
value = 'value'
the function should take these values as input and return a dictionary with sub-dictionaries inside and key,value added at the end dict.
Expected output = {'pma':{'device':{'clock':{'key':'value'}}}}
The challenge I am facing is, how to take the input string and define dynamic dictionaries with dict names as words in the string, while execution.
see this code
path = "['pma']['device']['clock']"
key = 'key'
value = 'value'
pl = path.replace('][', ',').replace(']', '').replace('[','').replace("'","").split(',')
res = {}
for item in reversed(pl):
if item == pl[-1]:
res[pl[-1]]={key:value}
else :
res = {item: res}
Output :
{'pma': {'device': {'clock': {'key': 'value'}}}}
The solution using re.findall() function and recursive makeDictFromPath() function:
import re
path = "['pma']['device']['clock']"
key = 'key'
val = 'value'
def makeDictFromPath(result, path):
result[path[0]] = {}
if len(path) > 1: return makeDictFromPath(result[path[0]], path[1:])
return result
result = {}
keys = re.findall(r'\[\'(\w+)\'\]', path)
makeDictFromPath(result, keys).get('clock', {})[key] = val
print(result)
The output:
{'pma': {'device': {'clock': {'key': 'value'}}}}
I am using below code snippet to get the red reduced price from amazon but simehow I am always getting the old price and not the reduced red one.
enter code here`def getSignedUrlAmazon(searchvalue):
params = {'ResponseGroup':'Medium',
'AssociateTag':'',
'Operation':'ItemSearch',
'SearchIndex':'All',
'Keywords':searchvalue}
action = 'GET'`enter code here`
server = "webservices.amazon.in"
path = "/onca/xml"
params['Version'] = '2011-08-01'
params['AWSAccessKeyId'] = AWS_ACCESS_KEY_ID
params['Service'] = 'AWSECommerceService'
params['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
# Now sort by keys and make the param string
key_values = [(urllib.quote(k), urllib.quote(v)) for k,v in params.items()]
key_values.sort()
# Combine key value pairs into a string.
paramstring = '&'.join(['%s=%s' % (k, v) for k, v in key_values])
urlstring = "http://" + server + path + "?" + \
('&'.join(['%s=%s' % (k, v) for k, v in key_values]))
# Add the method and path (always the same, how RESTy!) and get it ready to sign
hmac.update(action + "\n" + server + "\n" + path + "\n" + paramstring)
# Sign it up and make the url string
urlstring = urlstring + "&Signature="+\
urllib.quote(base64.encodestring(hmac.digest()).strip())
return urlstring
forgot the price grabbing part:
def getDataAmazon(searchvalue,PRICE, lock):
try:
searchvalue = removeStopWord(searchvalue)
url = getSignedUrlAmazon(searchvalue)
data = etree.parse(url)
root = objectify.fromstring(etree.tostring(data, pretty_print=True))
gd=[]
gd1=[]
counter = 0
if hasattr(root, 'Items') and hasattr(root.Items, 'Item'):
for item in root.Items.Item:
cd={}
priceCheckFlag = False
try :
if hasattr(item.ItemAttributes, 'EAN'):
cd['EAN'] = str(item.ItemAttributes.EAN)
elif hasattr(item, 'ASIN'):
cd['ASIN'] = str(item.ASIN)
cd['productName'] = str(item.ItemAttributes.Title)
if hasattr(item, 'SmallImage'):
cd['imgLink'] = str(item.SmallImage.URL)
elif hasattr(item, 'MediumImage'):
cd['imgLink'] = str(item.MediumImage.URL)
cd['numstar'] = "0"
cd['productdiv'] = '1'
cd['sellername'] = 'Amazon'
if hasattr(item, 'DetailPageURL'):
cd['visitstore'] = str(item.DetailPageURL)
if hasattr(item.ItemAttributes, 'ListPrice') and hasattr(item.ItemAttributes.ListPrice, 'Amount'):
cd['price'] = item.ItemAttributes.ListPrice.Amount/100.0
elif hasattr(item, 'OfferSummary') and hasattr(item.OfferSummary, 'LowestNewPrice'):
cd['price'] = item.OfferSummary.LowestNewPrice.Amount/100.0
I want to search something in unicode character like ŞÇİÖÜ
I define a seach function like that:
def search(request):
from django.db.models import Q
q = request.GET.get("search", "")
k = q.split()
list=[]
for i in k:
query = Q(title__icontains=i) | Q(content__icontains=i)
posts = Post.objects.filter(query).distinct().order_by('date')
list.append(posts)
total = 0
if list[0]:
total += len(list)
return render_to_response('search_results.html', RequestContext(request, {
"list": list,
"q": q,
"total": total,
}))
I have LANGUAGE_CODE = "UTF-8". When I want to search 'Şeker Öçü' it returns nothing ,although I have şeker öçü in posts.
The query string that you obtain from request.GET.get("search", "") is in ISO-8859-8, but Django's default encoding is UTF-8.
You will have to change either DEFAULT_CHARSET or HttpRequest.encoding to ISO-8859-8 to get the correct Unicode data. ( https://docs.djangoproject.com/en/dev/ref/settings/#default-charset )