I'm trying to write some variableas in a csv file.
Documentation (https://docs.python.org/2/library/csv.html) say the all input must be UTF-8 or ASCII, but I don't know how to set the encode ( I already tried .decode('utf-8'))
part of the code:
def get_events():
global SEVERITY, SEVERITY
get_host()
for HID,HNAME in zip(HOSTID,HOSTNAME):
EVENT = zapi.event.get(time_from=DATE_FROM,
time_till=DATE_TILL,
output='extend',
source='0',
hostids=HID,
select_acknowledges='extend',
)
for t in EVENT:
TRIGGERID = t['objectid']
TRIGGER = zapi.trigger.get(output='extend',
triggerids=TRIGGERID,
expandDescription='true',
)
for T in TRIGGER:
TRIGGER_D = T['description'].encode('utf-8')
SEVERITY = T['priority']
if int(SEVERITY) < 3 or TRIGGER_D.decode('utf-8') == 'Zabbix agent on %s is unreachable for 8 minutes' % HNAME or TRIGGER_D.decode('utf-8') == '%s jmx is not reachable' % HNAME:
continue
NS = t['ns']
HOUR = t['clock']
if t.get('value') == "0":
STATUS = "OK"
elif t.get('value')== "1":
STATUS = "PROBLEM"
else:
STATUS = "UNKNOWN"
if t['acknowledged'] == "1":
LISTA_DIC = t['acknowledges'][0]
USER = LISTA_DIC['alias']
MESSAGE = LISTA_DIC["message"]
else:
MESSAGE = ""
USER = ""
with open(FILE_OUT, 'wb') as FILE:
FILE = csv.writer(FILE,delimiter=';')
FILE.writerow((HNAME,STATUS,HOUR,NS,TRIGGER_D, SEVERITY,MESSAGE,USER)
I'm get the error
“UnicodeDecodeError: 'ascii' codec can't decode byte”
Another thing: How add a newline in
FILE.writerow((HNAME,STATUS,HOUR,NS,TRIGGER_D, SEVERITY,MESSAGE,USER))
..MESSAGE,USER "\n")) doesn't work
Can solve the problem change the encode to latin-1
FILE.writerow((HNAME,STATUS,HOUR,NS,TRIGGER_D.encode('latin-1'), SEVERITY,MESSAGE.encode('latin-1'),USER)
Related
I am creating an app where the server and clients run on the same machine, see picture.
I want the user to be able to send data from the server to a specific client (= specific window). For this, the user needs to know which ID belongs to which client (for example the corresponding ID could be displayed in each window's title).
Is it possible to get the corresponding descriptor ID on the client side? If not, how could I achieve the same result anyway?
I expect something like this as a result:
Here is an example code in pyside2 but I don't mind if the solution is using C++ qt.
QTCPServer:
import sys
from typing import List
from PySide2.QtCore import *
from PySide2.QtNetwork import *
from PySide2.QtWidgets import *
class MainWindow(QMainWindow):
new_message = Signal(bytes)
_connection_set: List[QTcpSocket] = []
def __init__(self):
super().__init__()
self.server = QTcpServer()
# layout
self.setWindowTitle("QTCPServer")
self._central_widget = QWidget()
self._main_layout = QVBoxLayout()
self.status_bar = QStatusBar()
self.text_browser_received_messages = QTextBrowser()
self._controller_layout = QHBoxLayout()
self.combobox_receiver = QComboBox()
self.lineEdit_message = QLineEdit()
self._controller_layout.addWidget(self.combobox_receiver)
self._controller_layout.addWidget(self.lineEdit_message)
self._buttons_layout = QHBoxLayout()
self.send_message_button = QPushButton("Send Message")
self.send_message_button.clicked.connect(self.send_message_button_clicked)
self._buttons_layout.addWidget(self.send_message_button)
# end layout
if self.server.listen(QHostAddress.Any, 8080):
self.new_message.connect(self.display_message)
self.server.newConnection.connect(self.new_connection)
self.status_bar.showMessage("Server is listening...")
else:
QMessageBox.critical(self, "QTCPServer", f"Unable to start the server: {self.server.errorString()}.")
self.server.close()
self.server.deleteLater()
sys.exit()
# set layout
self.setStatusBar(self.status_bar)
self.setCentralWidget(self._central_widget)
self._central_widget.setLayout(self._main_layout)
self._main_layout.addWidget(self.text_browser_received_messages)
self._main_layout.addLayout(self._controller_layout)
self._main_layout.addLayout(self._buttons_layout)
def new_connection(self) -> None:
while self.server.hasPendingConnections():
self.append_to_socket_list(self.server.nextPendingConnection())
def append_to_socket_list(self, socket: QTcpSocket):
self._connection_set.insert(len(self._connection_set), socket)
self.connect(socket, SIGNAL("readyRead()"), self.read_socket)
self.connect(socket, SIGNAL("disconnected()"), self.discard_socket)
self.combobox_receiver.addItem(str(socket.socketDescriptor()))
self.display_message(f"INFO :: Client with socket:{socket.socketDescriptor()} has just entered the room")
def read_socket(self):
socket: QTcpSocket = self.sender()
buffer = QByteArray()
socket_stream = QDataStream(socket)
socket_stream.setVersion(QDataStream.Qt_5_15)
socket_stream.startTransaction()
socket_stream >> buffer
if not socket_stream.commitTransaction():
message = f"{socket.socketDescriptor()} :: Waiting for more data to come.."
self.new_message.emit(message)
return
header = buffer.mid(0, 128)
file_type = header.split(",")[0].split(":")[1]
buffer = buffer.mid(128)
if file_type == "message":
message = f"{socket.socketDescriptor()} :: {str(buffer, 'utf-8')}"
self.new_message.emit(message)
def discard_socket(self):
socket: QTcpSocket = self.sender()
it = self._connection_set.index(socket)
if it != len(self._connection_set):
self.display_message(f"INFO :: A client has just left the room")
del self._connection_set[it]
socket.deleteLater()
self.refresh_combobox()
def send_message_button_clicked(self):
receiver = self.combobox_receiver.currentText()
if receiver == "Broadcast":
for socket in self._connection_set:
self.send_message(socket)
else:
for socket in self._connection_set:
if socket.socketDescriptor() == int(receiver):
self.send_message(socket)
return
self.lineEdit_message.clear()
def send_message(self, socket: QTcpSocket):
if not socket:
QMessageBox.critical(self, "QTCPServer", "Not connected")
return
if not socket.isOpen():
QMessageBox.critical(self, "QTCPServer", "Socket doesn't seem to be opened")
return
string = self.lineEdit_message.text()
socket_stream = QDataStream(socket)
socket_stream.setVersion(QDataStream.Qt_5_15)
header = QByteArray()
string_size = len(string.encode('utf-8'))
fstring = f"fileType:message,fileName:null,fileSize:{string_size}"
header.prepend(fstring.encode())
header.resize(128)
byte_array = QByteArray(string.encode())
byte_array.prepend(header)
socket_stream.setVersion(QDataStream.Qt_5_15)
socket_stream << byte_array
def display_message(self, string):
self.text_browser_received_messages.append(string)
def refresh_combobox(self):
self.combobox_receiver.clear()
self.combobox_receiver.addItem("Broadcast")
for socket in self._connection_set:
self.combobox_receiver.addItem(str(socket.socketDescriptor()))
if __name__ == '__main__':
app = QApplication(sys.argv)
w = MainWindow()
w.show()
sys.exit(app.exec_())
QTCPClient
import sys
from PySide2.QtCore import *
from PySide2.QtNetwork import *
from PySide2.QtWidgets import *
class MainWindow(QMainWindow):
new_message = Signal(bytes)
def __init__(self):
super().__init__()
self.socket = QTcpSocket(self)
# layout
self.setWindowTitle("QTCPClient")
self._central_widget = QWidget()
self._main_layout = QVBoxLayout()
self.status_bar = QStatusBar()
self.text_browser_received_messages = QTextBrowser()
self._controller_layout = QHBoxLayout()
self.lineEdit_message = QLineEdit()
self._controller_layout.addWidget(self.lineEdit_message)
self._buttons_layout = QHBoxLayout()
self.send_message_button = QPushButton("Send Message")
self.send_message_button.clicked.connect(self.on_send_message_button_clicked)
self._buttons_layout.addWidget(self.send_message_button)
# end layout
self.new_message.connect(self.display_message)
self.connect(self.socket, SIGNAL("readyRead()"), self.read_socket)
self.connect(self.socket, SIGNAL("disconnected()"), self.discard_socket)
# set layout
self.setStatusBar(self.status_bar)
self.setCentralWidget(self._central_widget)
self._central_widget.setLayout(self._main_layout)
self._main_layout.addWidget(self.text_browser_received_messages)
self._main_layout.addLayout(self._controller_layout)
self._main_layout.addLayout(self._buttons_layout)
self.socket.connectToHost(QHostAddress.LocalHost, 8080)
if self.socket.waitForConnected():
self.status_bar.showMessage("Connected to Server")
else:
QMessageBox.critical(self, "QTCPClient", f"The following error occurred: {self.socket.errorString()}.")
if self.socket.isOpen():
self.socket.close()
sys.exit()
def discard_socket(self):
self.socket.deleteLater()
self.socket = None
self.status_bar.showMessage("Disconnected!")
def read_socket(self):
buffer = QByteArray()
socket_stream = QDataStream(self.socket)
socket_stream.setVersion(QDataStream.Qt_5_15)
socket_stream.startTransaction()
socket_stream >> buffer
if not socket_stream.commitTransaction():
message = f"{self.socket.socketDescriptor()} :: Waiting for more data to come.."
self.new_message.emit(message)
return
header = buffer.mid(0, 128)
file_type = header.split(",")[0].split(":")[1]
buffer = buffer.mid(128)
if file_type == "message":
message = f"{self.socket.socketDescriptor()} :: {str(buffer, 'utf-8')}"
self.new_message.emit(message)
def on_send_message_button_clicked(self):
if not self.socket:
QMessageBox.critical(self, "QTCPServer", "Not connected")
return
if not self.socket.isOpen():
QMessageBox.critical(self, "QTCPServer", "Socket doesn't seem to be opened")
return
string = self.lineEdit_message.text()
socket_stream = QDataStream(self.socket)
socket_stream.setVersion(QDataStream.Qt_5_15)
header = QByteArray()
string_size = len(string.encode('utf-8'))
fstring = f"fileType:message,fileName:null,fileSize:{string_size}"
header.prepend(fstring.encode())
header.resize(128)
byte_array = QByteArray(string.encode())
byte_array.prepend(header)
socket_stream << byte_array
self.lineEdit_message.clear()
def display_message(self, string: str):
self.text_browser_received_messages.append(string)
if __name__ == '__main__':
app = QApplication(sys.argv)
w = MainWindow()
w.show()
sys.exit(app.exec_())
The socket descriptors are only valid for the constructor and they do not match on both sides.
One possibility is to automatically send a first "handshake" message to the client as soon as it's connected, the client will identify that message as a "descriptor id" type, and eventually set its window title.
In the following changes to your code, I'm using a simple fileType:descriptor header, and the descriptor id is actually sent as an integer value into the datastream. You can obviously use a string there, if you want to send any other value.
# server
def append_to_socket_list(self, socket: QTcpSocket):
# ...
descriptor = int(socket.socketDescriptor())
socket_stream = QDataStream(socket)
fstring = 'fileType:descriptor,fileName:null,fileSize:{},'.format(descriptor.bit_length())
header = QByteArray()
header.prepend(fstring.encode())
header.resize(128)
socket_stream << header
socket_stream.writeInt32(descriptor)
# client
def read_socket(self):
# ...
header = buffer.mid(0, 128)
fields = header.split(",")
file_type = fields[0].split(":")[1]
buffer = buffer.mid(128)
if file_type == "descriptor":
self.id = socket_stream.readInt32()
self.setWindowTitle("QTCPClient - id {}".format(self.id))
Some suggestions:
both signals have a bytes signature, but this is wrong as you're emitting those signals as str types; if you're not sure, you can use the basic object type;
the self.connect syntax is considered obsolete, use the "new" (well, not so new anymore) style one: object.signal.connect(slot); for instance:
self.socket.readyRead.connect(self.read_socket)
use QApplication.quit() instead of sys.exit(), so that the application properly does everything it needs before actually quitting the python interpreter;
instead of using the text value of the combo, you should use the user data:
descriptor = socket.socketDescriptor()
self.combobox_receiver.addItem(str(descriptor), descriptor)
then you can access it by using self.combobox_receiver.currentData() (you can add the "broadcast" item with a -1 value); you could even add the socket itself as user data;
to properly split the header without getting garbled results for the last field, you must add a final comma, otherwise split() will return the whole rest of the string;
Note for PyQt users: socketDescriptor() returns a sip.voidptr, to obtain the actual value use int(socket.socketDescriptor()).
The user wishes to have a database backup and restore functionality through the application. They want to be able to download the database file and upload it to restore the database whenever needed. The problem is that django is already running the current DB file. I wrote the following logic to restore the database.
folder ='./'
if request.method == 'POST':
myfile = request.FILES['file']
fs = FileSystemStorage(location=folder)
if myfile.name.split(".")[1] != "sqlite3":
return JsonResponse({"res":"Please upload a database file."})
if os.path.isfile(os.path.join(folder, "db.sqlite3")):
os.remove(os.path.join(folder, "db.sqlite3"))
filename = fs.save("db.sqlite3", myfile)
file_url = fs.url(filename)
return JsonResponse({"res":file_url})
I get the following error which is rightly justified:
[WinError 32] The process cannot access the file because it is being used by another process
So, is there a way I can achieve this functionality through my application?
I found a better way to create this functionality using a csv. One could store the data from the DB into a csv file and restore it when uploaded. Following is my implementation:
def back_up_done(request):
tables = getTableNames()
sql3_cursor = connection.cursor()
for table in tables:
sql3_cursor.execute(f'SELECT * FROM {table}')
with open('output.csv','a') as out_csv_file:
csv_out = csv.writer(out_csv_file)
for result in sql3_cursor:
csv_out.writerow('Null' if x is None else x for x in result)
csv_out.writerow("|")
BaseDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
csv_path = os.path.join(BaseDir, 'output.csv')
dbfile = File(open(csv_path, "rb"))
response = HttpResponse(dbfile, content_type='application/csv')
response['Content-Disposition'] = 'attachment; filename=%s' % 'backup.csv'
response['Content-Length'] = dbfile.size
os.remove(os.path.join("./", "output.csv"))
return response
def back_up_restore(request):
if request.method == 'POST':
DBfile = request.FILES['file']
cursor = connection.cursor()
if DBfile.name.split(".")[1] != "csv":
messages.add_message(request, messages.ERROR, "Please upload a CSV file.")
return redirect('back-up-db')
tables = getTableNames()
i = 0
deleteColumns()
decoded_file = DBfile.read().decode('utf-8').splitlines()
reader = csv.reader(decoded_file)
for row in reader:
if len(row) != 0:
if(row[0] == "|"):
i += 1
else:
query = f'''Insert into {tables[i]} values({concatRowValues(row)})'''
cursor.execute(query)
connection.close()
messages.add_message(request, messages.SUCCESS, "Data Restored Successfully.")
return redirect('login')
def concatRowValues(row):
data = ''
for i,value in enumerate(row):
if value == "False":
value = '0'
elif value == "True":
value = '1'
if i != len(row) - 1:
data = f'{data}{str(value)},' if value == "Null" else f'{data}\'{str(value)}\','
else:
data = f'{data}{str(value)}' if value == "Null" else f'{data}\'{str(value)}\''
return data
Where getTableNames and concatRowValues are helper functions to get the names of tables and to concatenate column values to form an executable sql statement, respectively. The "|" character is used to mark the end of a table's data.
" i am trying to read data from the serial connection and doing some stuff if it matches my string but its giving me errors when i close the serial connection port"
" for some reason i do not see this error if i use the serial.readline() method "
import time
import serial
from Queue import Queue
from threading import Thread
class NonBlocking:
def __init__(self, serial_connection, radio_serial_connection):
self._s = serial_connection
self._q = Queue()
self.buf = bytearray()
def _populateQueue(serial_connection, queue):
if type(serial_connection) == str:
return
self.s = serial_connection
while True:
i = self.buf.find(b"\n")
if i >= 0:
r = self.buf[:i + 1]
self.buf = self.buf[i + 1:]
queue.put(r)
while True:
i = max(1, min(2048, self.s.in_waiting))
data = self.s.read(i)
i = data.find(b"\n")
if i >= 0:
r = self.buf + data[:i + 1]
self.buf[0:] = data[i + 1:]
a = r.split('\r\n')
for item in a:
if item:
queue.put(item)
else:
self.buf.extend(data)
self._t = Thread(target=_populateQueue, args=(self._s, self._q))
self._t.daemon = True
self._t.start()
def read_all(self, timeout=None):
data = list()
if self._q.empty():
pass
while not self._q.empty():
data.append(self._q.get(block=timeout is not None, timeout=timeout))
return data
class SerialCommands:
def __init__(self, port, baudrate):
self.serial_connection = serial.Serial(port, baudrate)
self.queue_data = NonBlocking(self.serial_connection, '')
def read_data(self):
returned_info = self.queue_data.read_all()
return returned_info
def close_q(self):
self.serial_connection.close()
class qLibrary:
def __init__(self):
self.q = None
self.port = None
def close_q_connection(self):
self.q.close_q()
def establish_connection_to_q(self, port, baudrate=115200, delay=2):
self.delay = int(delay)
self.port = port
try:
if not self.q:
self.q = SerialCommands(self.port, int(baudrate))
except IOError:
raise AssertionError('Unable to open {0}'.format(port))
def verify_event(self, data, timeout=5):
timeout = int(timeout)
data = str(data)
# print data
while timeout:
try:
to_analyze = self.q.read_data()
for item in to_analyze:
print "item: ", item
if str(item).find(str(data)) > -1:
print "Found data: '{0}' in string: '{1}'".format(data, item)
except:
pass
time.sleep(1)
timeout -= 1
if __name__ == '__main__':
q1 = qLibrary()
q1.establish_connection_to_q('COM5')
q1.verify_event("ATE")
q1.close_q_connection()
" i expect the code to close the serial connection without any exceptions or errors "
the output is
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Python27\Lib\threading.py", line 801, in __bootstrap_inner
self.run()
File "C:\Python27\Lib\threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs)
File "C:/Program Files (x86)/serialtest1.py", >line 27, in _populateQueue
data = self.s.read(i)
File "C:\Program Files (x86)\venv\lib\site->packages\serial\serialwin32.py", line 283, in read
ctypes.byref(self._overlapped_read))
TypeError: byref() argument must be a ctypes instance, not 'NoneType'
If you define your serial port with no timeout it will get the default setting timeout=None which means when you call serial.read(x) the code will block until you read x bytes.
If you never get those x bytes your code will get stuck in there waiting forever, or at least until you receive more data on the buffer to get the total number of bytes received equal to x.
If you mix that up with threading, I'm afraid you are quite likely closing the port while you are trying to read.
You can probably fix this issue just defining a sensible read timeout on your port or changing the way you read. The general advice is to set a timeout that works for your application and read at least the maximum number of bytes you expect. Reading your code, that seems to be what you wanted to do. If so, you forgot to set the timeout.
If you have a reason not to set a timeout or you want to keep your reading routine as it is, you can make your code work if you cancel reading before closing. You can do that with serial.cancel_read()
i have a project with django .on the host when i want to upload an image sometime error occurred(problem with specific images)! the below show how i resize uploaded images:
def save_files_to_media(request, is_public=False, klass=None, conversation=None):
from apps.file.models import File
fs = FileSystemStorage()
file_items = {}
for data_item in request.data:
file_match = re.search('^fileToUpload\[(\d+)\]$', data_item)
if file_match and file_match.groups():
item_index = file_match.groups()[0]
if item_index not in file_items:
file_items[item_index] = {}
file_items[item_index]['file_to_upload'] = request.data[data_item]
else:
optimize_match = re.search('^optimizeType\[(\d+)\]$', data_item)
if optimize_match and optimize_match.groups():
item_index = optimize_match.groups()[0]
if item_index not in file_items:
file_items[item_index] = {}
file_items[item_index]['optimize_type'] = request.data[data_item]
files = []
for file_item_key in file_items:
input_file = file_items[file_item_key]['file_to_upload']
# TODO: checking validation. if input_file.name is not exist
optimize_type = file_items[file_item_key].get('optimize_type')
file_uuid = str(uuid4())
if is_public:
orig_filename, file_ext = splitext(basename(input_file.name))
directory_name = join(settings.MEDIA_ROOT, file_uuid)
filename = file_uuid + file_ext
else:
directory_name = join(settings.MEDIA_ROOT, file_uuid)
mkdir(directory_name)
filename = input_file.name
filepath = join(directory_name, filename)
fs.save(filepath, input_file)
is_optimized = False
if optimize_type == 'image':
is_success, filepath = image_optimizer(filepath)
filename = basename(filepath)
is_optimized = is_success
file_obj = File(
orig_name=filename,
uuid=file_uuid,
md5sum=get_md5sum(filepath),
filesize=get_filesize(filepath),
meta=get_meta_info(filepath),
is_optimized=is_optimized,
creator=request.user
)
if is_public:
file_obj.is_public = True
else:
file_obj.klass = klass
file_obj.conversation = conversation
file_obj.save()
files.append(file_obj)
return files
here is the error i got with some images:
unsupported Unicode escape sequence
LINE 1: ..., 'ada90ead20f7994837dced344266cc51', 145216, '', '{"FileTyp...
^
DETAIL: \u0000 cannot be converted to text.
CONTEXT: JSON data, line 1: ...ecTimeDigitized": 506779, "MakerNoteUnknownText":
its funny that in my local but not in host. for more information i must tell you guys my postgreSQL version is 11.3 and host postgreSQl is 9.5.17 . where you think is problem? as error it's seems for postgreSQL. thank you
I'm converting a HTML file into PDF in Django using Pisa. It is working when the content is only in English. But here the content will be in English and five other Indian languages(Tamil, Hindi, Telugu, Malayalam, and Kannada). I have given my code below.
views.py
def render_to_pdf1(template_src, context_dict):
template = get_template(template_src)
context = Context(context_dict)
html = template.render(context)
result = StringIO.StringIO()
pdf = pisa.pisaDocument(StringIO.StringIO(html.encode('UTF-8')), result)
return result.getvalue()
def print_pdf(request):
message = Message.objects.get(id = 1)
html_table_string = ''
html_table_string += '%s' % message.english
html_table_string += '%s' % message.tamil
html_table_string += '%s' % message.hindi
html_table_string += '%s' % message.telugu
html_table_string += '%s' % message.kannada
html_table_string += '%s' % message.malayalam
fileread = str(settings.TEMPLATE_DIRS[0])+str('/base_file.html')
fr = open(fileread, "r").read()
fr = fr.replace('message_content', html_table_string)
result = StringIO.StringIO()
filewrite = str(settings.TEMPLATE_DIRS[0]) + str('/temp_file.html')
empty = ""
fw = open(filewrite, 'w')
fw.write(empty)
fw.write(fr)
fw.close()
pdf_contents = render_to_pdf1('temp_file.html',result)
file_to_be_saved = ContentFile(pdf_contents)
name = (str(request.user.email) + ".pdf").replace("#", '')
pdf = Pdf.objects.create(name = name, user = request.user, created_by = request.user)
pdf.name.save(name ,file_to_be_saved)
file_path = Pdf.objects.get(user = request.user).name
pdf_file = str(file_path).split("media")[1]
return HttpResponseRedirect('/site_media' + pdf_file)
Here what I'm doing is:
Having a base template base_file.html.
Getting the message object by ID (ID will be dynamically supplied).
Then replacing the message_content with current content.
Writing it in a file temp_file.html.
Converting temp_file.html into a PDF file.
The converted PDF will be containing the message in English, Tamil, Hindi, Telugu, Kannada, and Malayalam. But I couldn't write the other language contents in the HTML file and couldn't convert it. The error I'm getting is 'ascii' codec can't encode characters in position 1066-1075: ordinal not in range(128) and occurs in the line fw.write(fr).
So how can I achieve this? I want to print the PDF file with content in all these languages.