\u0000 cannot be converted to text in django/postgreSQl - django

i have a project with django .on the host when i want to upload an image sometime error occurred(problem with specific images)! the below show how i resize uploaded images:
def save_files_to_media(request, is_public=False, klass=None, conversation=None):
from apps.file.models import File
fs = FileSystemStorage()
file_items = {}
for data_item in request.data:
file_match = re.search('^fileToUpload\[(\d+)\]$', data_item)
if file_match and file_match.groups():
item_index = file_match.groups()[0]
if item_index not in file_items:
file_items[item_index] = {}
file_items[item_index]['file_to_upload'] = request.data[data_item]
else:
optimize_match = re.search('^optimizeType\[(\d+)\]$', data_item)
if optimize_match and optimize_match.groups():
item_index = optimize_match.groups()[0]
if item_index not in file_items:
file_items[item_index] = {}
file_items[item_index]['optimize_type'] = request.data[data_item]
files = []
for file_item_key in file_items:
input_file = file_items[file_item_key]['file_to_upload']
# TODO: checking validation. if input_file.name is not exist
optimize_type = file_items[file_item_key].get('optimize_type')
file_uuid = str(uuid4())
if is_public:
orig_filename, file_ext = splitext(basename(input_file.name))
directory_name = join(settings.MEDIA_ROOT, file_uuid)
filename = file_uuid + file_ext
else:
directory_name = join(settings.MEDIA_ROOT, file_uuid)
mkdir(directory_name)
filename = input_file.name
filepath = join(directory_name, filename)
fs.save(filepath, input_file)
is_optimized = False
if optimize_type == 'image':
is_success, filepath = image_optimizer(filepath)
filename = basename(filepath)
is_optimized = is_success
file_obj = File(
orig_name=filename,
uuid=file_uuid,
md5sum=get_md5sum(filepath),
filesize=get_filesize(filepath),
meta=get_meta_info(filepath),
is_optimized=is_optimized,
creator=request.user
)
if is_public:
file_obj.is_public = True
else:
file_obj.klass = klass
file_obj.conversation = conversation
file_obj.save()
files.append(file_obj)
return files
here is the error i got with some images:
unsupported Unicode escape sequence
LINE 1: ..., 'ada90ead20f7994837dced344266cc51', 145216, '', '{"FileTyp...
^
DETAIL: \u0000 cannot be converted to text.
CONTEXT: JSON data, line 1: ...ecTimeDigitized": 506779, "MakerNoteUnknownText":
its funny that in my local but not in host. for more information i must tell you guys my postgreSQL version is 11.3 and host postgreSQl is 9.5.17 . where you think is problem? as error it's seems for postgreSQL. thank you

Related

How to restore sqlite3 database file programmatically in django

The user wishes to have a database backup and restore functionality through the application. They want to be able to download the database file and upload it to restore the database whenever needed. The problem is that django is already running the current DB file. I wrote the following logic to restore the database.
folder ='./'
if request.method == 'POST':
myfile = request.FILES['file']
fs = FileSystemStorage(location=folder)
if myfile.name.split(".")[1] != "sqlite3":
return JsonResponse({"res":"Please upload a database file."})
if os.path.isfile(os.path.join(folder, "db.sqlite3")):
os.remove(os.path.join(folder, "db.sqlite3"))
filename = fs.save("db.sqlite3", myfile)
file_url = fs.url(filename)
return JsonResponse({"res":file_url})
I get the following error which is rightly justified:
[WinError 32] The process cannot access the file because it is being used by another process
So, is there a way I can achieve this functionality through my application?
I found a better way to create this functionality using a csv. One could store the data from the DB into a csv file and restore it when uploaded. Following is my implementation:
def back_up_done(request):
tables = getTableNames()
sql3_cursor = connection.cursor()
for table in tables:
sql3_cursor.execute(f'SELECT * FROM {table}')
with open('output.csv','a') as out_csv_file:
csv_out = csv.writer(out_csv_file)
for result in sql3_cursor:
csv_out.writerow('Null' if x is None else x for x in result)
csv_out.writerow("|")
BaseDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
csv_path = os.path.join(BaseDir, 'output.csv')
dbfile = File(open(csv_path, "rb"))
response = HttpResponse(dbfile, content_type='application/csv')
response['Content-Disposition'] = 'attachment; filename=%s' % 'backup.csv'
response['Content-Length'] = dbfile.size
os.remove(os.path.join("./", "output.csv"))
return response
def back_up_restore(request):
if request.method == 'POST':
DBfile = request.FILES['file']
cursor = connection.cursor()
if DBfile.name.split(".")[1] != "csv":
messages.add_message(request, messages.ERROR, "Please upload a CSV file.")
return redirect('back-up-db')
tables = getTableNames()
i = 0
deleteColumns()
decoded_file = DBfile.read().decode('utf-8').splitlines()
reader = csv.reader(decoded_file)
for row in reader:
if len(row) != 0:
if(row[0] == "|"):
i += 1
else:
query = f'''Insert into {tables[i]} values({concatRowValues(row)})'''
cursor.execute(query)
connection.close()
messages.add_message(request, messages.SUCCESS, "Data Restored Successfully.")
return redirect('login')
def concatRowValues(row):
data = ''
for i,value in enumerate(row):
if value == "False":
value = '0'
elif value == "True":
value = '1'
if i != len(row) - 1:
data = f'{data}{str(value)},' if value == "Null" else f'{data}\'{str(value)}\','
else:
data = f'{data}{str(value)}' if value == "Null" else f'{data}\'{str(value)}\''
return data
Where getTableNames and concatRowValues are helper functions to get the names of tables and to concatenate column values to form an executable sql statement, respectively. The "|" character is used to mark the end of a table's data.

django handle slowly when doing Stress Testing

i had built django server to handle some work.And when i sent 10000 requests to django,django will became slow after 4000th requests,it will report socket timeout randomly.And i take this one data to test it alone,it is ok.So,how can i fix this bug?
i have no idea to fix it
def handle(request) :
sys.stderr.write( "[info]request come in {} \n".format(sys._getframe().f_code.co_name))
ret_dict = dict()
ret_str = ""
if ("vid" in request.GET) and ("title" in request.GET):
id = request.GET["id"]
title = urllib.parse.unquote(request.GET["title"],encoding ='utf-8')
title = request.GET["title"]
req_video = dict()
req_video["vid"] = vid
req_video["title"] = title
tags = handle_function(req_video)
ret_dict["vid"] = vid
ret_dict["tags"] = tags
ret_str = json.dumps(ret_dict)
return HttpResponse(ret_str)
client codes:
for line in sys.stdin:
line = line.rstrip('\n')
fields = line.split("\t")
vid = fields[0]
title = fields[1]
page_params_mq = {'vid':vid,'title':title}
try :
rec_mq_res = get_rec_mingqiang(server_ip_mq, server_port_mq, web_page_mq, page_params_mq)
except:
print (line)
continue
vid = rec_mq_res["vid"]
tags = rec_mq_res["tags"]
fo.write(vid+"\t"+tags+"\n")
cat test_data|python client.py, there are 10000 rows in test_data,and it send request sequentially,and will be hunt randomly after 4000.
django can handle these requests quickly

Regular expression to find function definitions in Python file

I have a python project with function definitions written in CamelCase style.
I'm trying to write a script to convert them to snake_case style.
CaseFormatter class
import re
class CaseFormatter:
def __init__(self, file_directory):
self.first_cap_re = re.compile('(.)([A-Z][a-z]+)')
self.all_cap_re = re.compile('([a-z0-9])([A-Z])')
self.functions_dict = {}
self.file_directory = file_directory
self.file = open(file_directory, "r", encoding="UTF-8")
self.file_content = self.file.read()
self.names_dictionary = {}
def convert_camel_case_to_snake_case(self, name):
"""this function convert a camel case name to a snake case name """
s1 = self.first_cap_re.sub(r'\1_\2', name)
self.names_dictionary[name] = self.all_cap_re.sub(r'\1_\2', s1).lower()
def find_camel_case_functions(self):
"""this function finds camel case functions in a file """
s1 = re.findall("^\s*def (\S+)\s*\(\s*\S+\s*(?:,\s*\S+)*\):$", self.file_content)
return s1
def replace_functions(self):
# file_content = open(self.file_directory, "r", encoding="UTF-8").read()
self.file_content = self.file_content.replace(";", "")
for key, val in self.names_dictionary.items():
self.file_content = self.file_content.replace(key, val)
# print(self.file_content)
self.file = open(self.file_directory, "w", encoding="UTF-8")
print(self.file_content)
self.file.write(self.file_content)
testing the CaseFormatter class
import os
from CaseFormatter import *
walk_dir = 'some dirctory'
print('walk_dir = ' + walk_dir)
print('walk_dir (absolute) = ' + os.path.abspath(walk_dir))
for root, subDirs, files in os.walk(walk_dir):
print('--\nroot = ' + root)
for filename in files:
file_path = os.path.join(root, filename)
if filename.endswith('.py') and filename != "__init__.py":
print('\t- file %s (full path: %s)' % (filename, file_path))
case_formatter = CaseFormatter(file_path)
# print(case_formatter.find_camel_case_functions())
for function_name in case_formatter.find_camel_case_functions():
case_formatter.convert_camel_case_to_snake_case(function_name)
print(case_formatter.names_dictionary)
case_formatter.replace_functions()
I found the RegEx to find function definitions here.
When I tried it on my project it gave me no results, the RegEx didn't work as I think.
As an example of one of the files in the project:
class UnvoweledPattern(object):
String = ''
Rules = []
IDs = []
def __init__(self, string, rules, ids):
self.String = string
self.Rules = rules
self.IDs = ids
pass
def GetRootsStringsAndRules(self, string):
if (string == None):
string = self.String
rootStrings = []
rootRules = []
for j in range(len(self.Rules)):
rootRule = ''
rootString = ''
for k in range(len(self.Rules[j])):
rootRule += self.Rules[j][k]
if self.Rules[j][k].isdigit():
rootString += string[int(self.Rules[j][k]) - 1]
else:
rootString += self.Rules[j][k]
rootStrings.append(rootString)
rootRules.append(rootRule)
return [rootStrings, rootRules]

Get SOAP attachment

There is a lot of questions with same subject, but no replies, especially about receiving. There exist example how to send attachment, but I didn't found how to receive it.
Is there any solution on python for receiving attachments? I even agree to change my SOAP tool from suds to anything that will works.
Thank you in advance.
I solved it with suds.
def GetWithFile(self, client, servicename, modelthings):
func = client.get_suds_func('Retrieve' + servicename)
clientclass = func.clientclass({})
SoapClient = clientclass(func.client, func.method)
binding = func.method.binding.input
soap_xml = binding.get_message(func.method, [modelthings], {})
soap_xml.children[0].children[1].children[0].attributes.append(u'attachmentInfo="true"')
soap_xml.children[0].children[1].children[0].attributes.append(u'attachmentData="true"')
soap_xml.children[0].children[1].children[0].attributes.append(u'ignoreEmptyElements="true"')
SoapClient.last_sent(soap_xml)
plugins = PluginContainer(SoapClient.options.plugins)
plugins.message.marshalled(envelope=soap_xml.root())
if SoapClient.options.prettyxml:
soap_xml = soap_xml.str()
else:
soap_xml = soap_xml.plain()
soap_xml = soap_xml.encode('utf-8')
plugins.message.sending(envelope=soap_xml)
request = Request(SoapClient.location(), soap_xml)
request.headers = SoapClient.headers()
reply = SoapClient.options.transport.send(request)
print(reply)
Files = []
boundary = self.find_substring(reply.headers['content-type'], 'boundary="', '"')
if boundary is not "":
list_of_data = reply.message.split(boundary)
list_of_data.pop(0)
list_of_data.pop(len(list_of_data) - 1)
soap_body = '<SOAP-ENV:Envelope' + self.find_substring(list_of_data[0], '<SOAP-ENV:Envelope', '</SOAP-ENV:Envelope>') + '</SOAP-ENV:Envelope>'
for line in list_of_data[1:]:
File = SMFile()
Files.append(File)
File.filename = self.find_substring(line, 'Content-Location: ', '\r\n')
File.key = self.find_substring(line, 'Content-ID: ', '\r\n')
idx = line.index( 'Content-ID:' )
start_idx = line.index( '\r\n\r\n' , idx ) + len('\r\n\r\n')
fin_idx = line.rindex( '\r\n--', start_idx )
File.body = line[start_idx: fin_idx]
File.size = fin_idx - start_idx
else:
soap_body = '<SOAP-ENV:Envelope' + self.find_substring(reply.message, '<SOAP-ENV:Envelope', '</SOAP-ENV:Envelope>') + '</SOAP-ENV:Envelope>'
ctx = plugins.message.received(reply=soap_body)
soap_body = ctx.reply
if SoapClient.options.retxml:
answer = soap_body
else:
answer = SoapClient.succeeded(binding, soap_body)
dict = {}
self.FieldsToDict(answer.model.instance, dict)
return {u'body': answer, u'Files': Files}
Here we extract some low level of suds, being able to fix any field in envelope. Then, after reply was got, we parse all boundaries and receive as many files, as we got.

django - HTML to PDF in Indian languages with pisa

I'm converting a HTML file into PDF in Django using Pisa. It is working when the content is only in English. But here the content will be in English and five other Indian languages(Tamil, Hindi, Telugu, Malayalam, and Kannada). I have given my code below.
views.py
def render_to_pdf1(template_src, context_dict):
template = get_template(template_src)
context = Context(context_dict)
html = template.render(context)
result = StringIO.StringIO()
pdf = pisa.pisaDocument(StringIO.StringIO(html.encode('UTF-8')), result)
return result.getvalue()
def print_pdf(request):
message = Message.objects.get(id = 1)
html_table_string = ''
html_table_string += '%s' % message.english
html_table_string += '%s' % message.tamil
html_table_string += '%s' % message.hindi
html_table_string += '%s' % message.telugu
html_table_string += '%s' % message.kannada
html_table_string += '%s' % message.malayalam
fileread = str(settings.TEMPLATE_DIRS[0])+str('/base_file.html')
fr = open(fileread, "r").read()
fr = fr.replace('message_content', html_table_string)
result = StringIO.StringIO()
filewrite = str(settings.TEMPLATE_DIRS[0]) + str('/temp_file.html')
empty = ""
fw = open(filewrite, 'w')
fw.write(empty)
fw.write(fr)
fw.close()
pdf_contents = render_to_pdf1('temp_file.html',result)
file_to_be_saved = ContentFile(pdf_contents)
name = (str(request.user.email) + ".pdf").replace("#", '')
pdf = Pdf.objects.create(name = name, user = request.user, created_by = request.user)
pdf.name.save(name ,file_to_be_saved)
file_path = Pdf.objects.get(user = request.user).name
pdf_file = str(file_path).split("media")[1]
return HttpResponseRedirect('/site_media' + pdf_file)
Here what I'm doing is:
Having a base template base_file.html.
Getting the message object by ID (ID will be dynamically supplied).
Then replacing the message_content with current content.
Writing it in a file temp_file.html.
Converting temp_file.html into a PDF file.
The converted PDF will be containing the message in English, Tamil, Hindi, Telugu, Kannada, and Malayalam. But I couldn't write the other language contents in the HTML file and couldn't convert it. The error I'm getting is 'ascii' codec can't encode characters in position 1066-1075: ordinal not in range(128) and occurs in the line fw.write(fr).
So how can I achieve this? I want to print the PDF file with content in all these languages.