Please advise how can I keep order while using multiprocessing in Python. Both func and somefunc return lists. Thanks,
results = []
def callback(*args): results.append(*args)
def in_parallel(fn, func):
print 'Process started on',time.strftime('%H:%M:%S')
pool = mp.Pool(processes = 4)
for num, i in enumerate(somefunc(fn)):
pool.apply_async(func, args = (i,), callback = callback([num, callback]))
pool.close()
pool.join()
return results
The problem has been solved with the following:
r = pool.map_async(...)
r.wait()
Related
Once a single function needs to be awaited, it seems like a cascading of functions that need to be awaited, until you reach a point where adding await breaks stuff:
I have dash code that intermittently gets called:
def get_data(symbolStr):
print("Getting AR for", symbolStr)
df = amp.get_data_job(symbolStr) #calls async code and need to be awaited
return df
...
html.Div(
dcc.Interval(id='interval1', interval = 900 * 1000, n_intervals=0),
),
html.Div(
dash_table.DataTable(
id='table',
columns=[{"name": i, "id": i} for i in magic_columns],
editable=True,
row_deletable=True
),
)
#app.callback(
[dash.dependencies.Output('table', 'data')],
[dash.dependencies.Input('interval1', 'n_intervals')],
[dash.dependencies.State('table', 'data')]
)
def update_interval(n, data):
allSymbolsDf = pd.DataFrame()
for symbolStr in symbolsStr:
df = get_data(symbolStr)
allSymbolsDf = allSymbolsDf.append(df)
time.sleep(3)
return [allSymbolsDf.to_dict('rows')]
Thing is, get_data is a function that calls asynchronous code
df = amp.get_data_job(symbolStr)
I can't turn the callback update_interval into an asynchronous function (AFAIK). What is the recommended way out of this. I mean, still keeping dash.
Ran into the same situation using FastApi and serving dash on the side. Use asyncio.run()
import asyncio
async def asyn_fun(foo):
return True if foo == "bar" else False
def sync_fun(foo):
return asyncio.run(asyn_fun(foo))
print(sync_fun("bar"))
I am trying to send a file across the network using Twisted with the LineReceiver protocol. The issue I am seeing is that when I read a binary file and try to send the chunks they simply don't send.
I am reading the file using:
import json
import time
import threading
from twisted.internet import reactor, threads
from twisted.protocols.basic import LineReceiver
from twisted.internet import protocol
MaximumMsgSize = 15500
trySend = True
connectionToServer = None
class ClientInterfaceFactory(protocol.Factory):
def buildProtocol(self, addr):
return WoosterInterfaceProtocol(self._msgProcessor, self._logger)
class ClientInterfaceProtocol(LineReceiver):
def connectionMade(self):
connectionToServer = self
def _DecodeMessage(self, rawMsg):
header, body = json.loads(rawMsg)
return (header, json.loads(body))
def ProcessIncomingMsg(self, rawMsg, connObject):
# Decode raw message.
decodedMsg = self._DecodeMessage(rawMsg)
self.ProccessTransmitJobToNode(decodedMsg, connObject)
def _BuildMessage(self, id, msgBody = {}):
msgs = []
fullMsgBody = json.dumps(msgBody)
msgBodyLength = len(fullMsgBody)
totalParts = 1 if msgBodyLength <= MaximumMsgSize else \
int(math.ceil(msgBodyLength / MaximumMsgSize))
startPoint = 0
msgBodyPos = 0
for partNo in range(totalParts):
msgBodyPos = (partNo + 1) * MaximumMsgSize
header = {'ID' : id, 'MsgParts' : totalParts,
'MsgPart' : partNo }
msg = (header, fullMsgBody[startPoint:msgBodyPos])
jsonMsg = json.dumps(msg)
msgs.append(jsonMsg)
startPoint = msgBodyPos
return (msgs, '')
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
for filename in exportedFiles:
with open (filename, "rb") as exportFileHandle:
data = exportFileHandle.read().encode('base64')
params = {
ExportFileToMaster_Tag.TaskID : taskID,
ExportFileToMaster_Tag.FileContents : data,
ExportFileToMaster_Tag.Filename : filename
}
msgs, _ = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
connection.sendLine(m)
def lineReceived(self, data):
threads.deferToThread(self.ProcessIncomingMsg, data, self)
def ConnectFailed(reason):
print 'Connection failed..'
reactor.callLater(20, reactor.callFromThread, ConnectToServer)
def ConnectToServer():
print 'Connecting...'
from twisted.internet.endpoints import TCP4ClientEndpoint
endpoint = TCP4ClientEndpoint(reactor, 'localhost', 8181)
deferItem = endpoint.connect(factory)
deferItem.addErrback(ConnectFailed)
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
reactor.callFromThread(ConnectToServer)
factory = ClientInterfaceFactory()
protocol = ClientInterfaceProtocol()
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
Is there something I am doing wrong?file is sent, it's when the write is multi part or there are more than one file it struggles.
If a single write occurs then the m
Note: I have updated the question with a crude piece of sample code in the hope it makes sense.
_BuildMessage returns a two-tuple: (msgs, '').
Your network code iterates over this:
msgs = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
So your network code first tries to send a list of json encoded data and then tries to send the empty string. It most likely raises an exception because you cannot send a list of anything using sendLine. If you aren't seeing the exception, you've forgotten to enable logging. You should always enable logging so you can see any exceptions that occur.
Also, you're using time.sleep and you shouldn't do this in a Twisted-based program. If you're doing this to try to avoid overloading the receiver, you should use TCP's native backpressure instead by registering a producer which can receive pause and resume notifications. Regardless, time.sleep (and your loop over all the data) will block the entire reactor thread and prevent any progress from being made. The consequence is that most of the data will be buffered locally before being sent.
Also, your code calls LineReceiver.sendLine from a non-reactor thread. This has undefined results but you can probably count on it to not work.
This loop runs in the main thread:
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
while the reactor runs in another thread:
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
ProcessTransmitJobToNode simply calls self.sendLine:
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
You should probably remove the use of threading entirely from the application. Time-based events are better managed using reactor.callLater (your main-thread loop effectively generates a call to ProcessTransmitJobToNode once hundred times a second (modulo effects of the trySend flag)).
You may also want to take a look at https://github.com/twisted/tubes as a better way to manage large amounts of data with Twisted.
I just ran into some strange behavior that has me stumped. I'm writing a simple little GUI for some in-house data processing. I want to allow a user to switch between a few different data-processing modes and input some parameters which define how the data is processed for each mode. The problem is that when the user inputs new parameters, the app ignores requests to switch modes.
The code below replicates the issue. I apologize for the size, this was the shortest code that replicates the problem.
import Tkinter as Tk
class foo(Tk.Frame):
def __init__(self):
self.master = master =Tk.Tk()
Tk.Frame.__init__(self,self.master) #Bootstrap
#Here mode and parameters as key, value pairs
self.data = {'a':'Yay',
'b':'Boo'
}
self.tex = Tk.Text(master=master)
self.tex.grid(row=0,column=0,rowspan=3,columnspan=4)
self.e = Tk.Entry(master=master)
self.e.grid(row=3,column=0,columnspan=4)
self.sv =Tk.StringVar()
self.sv.set('a')
self.b1 = Tk.Radiobutton(master=master,
text = 'a',
indicatoron = 0,
variable = self.sv,
value = 'a')
self.b2 = Tk.Radiobutton(master=master,
text = 'b',
indicatoron = 0,
variable = self.sv,
value = 'b')
self.b3 = Tk.Button(master = master,
text='Apply',command=self.Apply_Func)
self.b4 = Tk.Button(master = master,
text='Print',command=self.Print_Func)
self.b1.grid(row=4,column=0)
self.b2.grid(row=4,column=1)
self.b3.grid(row=4,column=2)
self.b4.grid(row=4,column=3)
def Apply_Func(self):
self.innerdata = self.e.get()
def Print_Func(self):
self.tex.insert(Tk.END,str(self.innerdata)+'\n')
#This is how I'm retrieving the user selected parameters
#property
def innerdata(self):
return self.data[self.sv.get()]
#And how I'm setting the user defined parameters
#innerdata.setter
def innerdata(self,value):
self.data[self.sv.get()] = value
if __name__ == "__main__":
app = foo()
app.mainloop()
Expected behavior:
1) Press button 'a' then 'print' prints:
Yay
2) Press button 'b' then 'print' prints:
Boo
3) Type 'Zep Rocks' into the entry field and press apply
4) Pressing 'print' now yields
Zep Rocks
5) Pressing 'a' then 'print' should yield
Yay
But instead yields
Zep Rocks
Which might be true, but not desired right now. What is going on here?
Edit: I have some new information. Tk.Frame in python 2.7 is not a new-style class. It isn't friendly with descriptors, so rather than interpreting the '=' as a request to use the foo.innerdata's __set__ method, it just assigns the result of self.e.get() to innerdata.
ARGLEBARGLE!!!
Still an open question: how do I get this to do what I want in a clean manner?
So the core problem is that Tk.Frame doesn't subclass from object, so it is not a new-style python class. Which means it doesn't get down with descriptors like I was trying to use. One solution that I found is to subclass my app from object instead.
Code that solves my problem is below:
import Tkinter as Tk
class foo(object):
def __init__(self,master):
self.master = master #Bootstrap
self.mainloop = master.mainloop
self.data = {'a':{'value':7,'metavalue':False},
'b':{'value':'Beeswax','metavalue':True}
}
self.tex = Tk.Text(master=master)
self.tex.grid(row=0,column=0,rowspan=3,columnspan=4)
self.e = Tk.Entry(master=master)
self.e.grid(row=3,column=0,columnspan=4)
self.sv =Tk.StringVar()
self.sv.set('a')
self.b1 = Tk.Radiobutton(master=master,
text = 'a',
indicatoron = 0,
variable = self.sv,
value = 'a')
self.b2 = Tk.Radiobutton(master=master,
text = 'b',
indicatoron = 0,
variable = self.sv,
value = 'b')
self.b3 = Tk.Button(master = master,text='Apply',command=self.Apply_Func)
self.b4 = Tk.Button(master = master,text='Print',command=self.Print_Func)
self.b1.grid(row=4,column=0)
self.b2.grid(row=4,column=1)
self.b3.grid(row=4,column=2)
self.b4.grid(row=4,column=3)
def Apply_Func(self):
self.innerdata = self.e.get()
def Print_Func(self):
self.tex.insert(Tk.END,str(self.innerdata)+'\n')
#property
def innerdata(self):
return self.data[self.sv.get()]
#innerdata.setter
def innerdata(self,value):
self.data[self.sv.get()] = value
if __name__ == "__main__":
master = Tk.Tk()
app = foo(master)
app.mainloop()
I have the code:
import time
import numpy as np
from scipy.optimize import fmin_tnc
from enthought.traits.api import *
from enthought.traits.ui.api import *
class Minimizer(HasTraits):
iteration = Int(0)
run = Button
def callback(self, x):
self.iteration += 1
print self.iteration
time.sleep(0.5)
def func(self, x):
return (x**2).sum()
def fprime(self, x):
return 2*x
def minimize(self):
x0 = np.random.rand(50)
fmin_tnc(self.func, x0, fprime=self.fprime, messages=0, callback = self.callback)
def _run_fired(self):
self.minimize()
traits_view = View(Item('iteration'), UItem('run'))
m = Minimizer()
m.configure_traits()
After running the above and pressing Run button i expected the 'iteration' attribute will be updated in the GUI at each iteration, but this is not the case. I suspect that this is because this value is changed by callback from C. What should be done to update the user interface in these circumstances?
Regards,
Marek
The call to m.configure_traits() is blocking, which means execution of your script will not continue past that line until you close the window created by that call. In other words, m.minimize does not get called while the window is open.
I found the solution. Simply, 'minimize' method have to be non-blocking, so implementing minimization in separate thread, like this:
def minimize(self):
x0 = np.random.rand(50)
#fmin_tnc(self.func, x0, fprime=self.fprime, messages=0, callback = self.callback)
import thread
thread.start_new_thread(fmin_tnc, (self.func, x0), {'fprime':self.fprime, 'messages':0, 'callback':self.callback})
will result in updating the UI at real-time...
Thanks,
Marek
I am trying to invoke a method from combo box selected change event
with lambda expression but I am stuck with following error
TypeError: () takes no arguments (1 given)
I think I have passed 1 argument as per the method definition, could somebody please help me where I am wrong
or any other combobox selected change event code will be great help!
please note my code
self.boxWidget[boxName].bind("<<ComboboxSelected>>", lambda:invoke_Setting_Group(self))
def invoke_My_method1(self):
print "expand another window"
I am trying to pass the first class object to the second python script file for variable value assigning easeness.I tried to use this combox event change code without lambda then I noticed that this method is getting called automatically so I used lambda to prevent this automatic method calling
Sorry I am not having the knowledge on lambda expression usage; here I used only to prevent the automatic method execution. Without lambda expression I noticed my combo box function starts automatically, I did not understand why it happens so?
I am using TKinter python 2.6
More Detailed Code of above:
#Main_GUI_Class.py
##----------------------
import sys
class App():
def __init__ (self,master,geometry=None,root=None):
try:
self.master=master
if not root:
self.root=Tkinter.Toplevel(master)
def initUI(self):
try:
self.master.title("GUI")
menubar = Menu(self.master)
self.root.config(menu=menubar)
fileMenu.add_command(label='Open')
submenu_ncss.add_command(label='Model Setting',command=lambda:Combo_Expand_Script.Call_Model_Setting(self))
##----------------------
def main():
r = Tkinter.Tk()
r.withdraw()
r.title("GUI Sample")
r.wm_iconbitmap(Pic1)
v = App(r)
r.mainloop()
if __name__ == "__main__":
main()
##Combo_Expand_Script.py
##-----------------------
import sys
import Tkinter
import Main_GUI_Class
def Call_Model_Setting(self):
try:
self.PopUpWin = Toplevel(bg='#54596d',height=500, width=365)
self.PopUpWin.title("POP UP SETTING")
#Combo Boxs in Pop Up
boxNameGroup="boxSetting"
boxPlaceY=0
for Y in range(4):
boxName=boxNameGroup+str(Y)
if Y == 0:
boxPlaceY=50
else:
boxPlaceY=boxPlaceY+40
self.box_value = StringVar()
self.boxWidget[boxName] = ttk.Combobox(self.PopUpWin, height=1, width=20)
if Y== 0:
self.boxWidget[boxName]['values'] = ('A', 'B')
self.boxWidget[boxName].current(1)
if Y== 1:
self.boxWidget[boxName]['values'] = ('X', 'Y')
self.boxWidget[boxName].bind("<<ComboboxSelected>>",lambda:invoke_Setting_Group(self))
self.boxWidget[boxName].place(x=180, y = boxPlaceY)
#Buttons in Pop Up
self.btnApply = tk.Button(self.PopUpWin,width=10, height=1,text="Apply",relief=FLAT,bg=btn_Bg_Color,command=lambda: treeDataTransfer(self,0))
self.btnApply.pack()
self.btnApply.place(x=75, y = 460)
self.btnCancel = tk.Button(self.PopUpWin,width=10, height=1,text="Cancel",relief=FLAT,command=lambda: deleteTreeNodes(self))
self.btnCancel.pack()
self.btnCancel.place(x=170, y = 460)
except IOError:
print "Error: data error"
def invoke_Setting_Group(self):#, event=None
try:
#self.boxName.current(0)
self.boxWidget["boxSetting3"].current(0)
self.PopUpWin['width']=1050
self.PopUpWin['height']=700
self.btnApply.place(x=500, y = 550)
self.btnCancel.place(x=600, y = 550)
self.txtWidget={}
lsttxtSetting = ['1', '2','3 ','4','5 ','6','7','8','9','10']
for t in range(10):
txtName=txtNameGroupTS+str(t)
if t == 0:
txtPlaceY=120
else:
txtPlaceY=txtPlaceY+30
self.txtWidget[txtName] = Text(self.groupSettingFrame,height=1, width=10,borderwidth = 2)
self.txtWidget[txtName].insert(INSERT, lsttxtSetting[t])
self.txtWidget[txtName].pack()
self.txtWidget[txtName].place(x=200, y = txtPlaceY)
except IOError:
print "Error: Group Settings Popup error"
def turbDataTransferBind(self):
for P in range(0,3):
boxName="boxSetting"+str(X)
dataSettingbox=self.lstTurb[X]+" "+self.boxWidget[boxName].get()
self.root_node_Setting = self.tree.insert( self.root_node_ChildSetting["ChildSettingNode"], 'end', text=dataSettingbox, open=True)
def treeDataTransfer(self,dlgTurbFlag):
self.treeDataTransferBind()
print "data tranfer sucess"
def deleteTreeNodes(self):
print "delete nodes"
command= and bind expect function name - without () and arguments - so in place of
If you use
.bind("<<ComboboxSelected>>", invoke_Setting_Group(self) )
then you use result from invoke_Setting_Group(self) as second argument in .bind(). This way you could dynamicly generate function used as argument in bind
TypeError: () takes no arguments (1 given)
This means you have function function() but python run it as function(arg1)
You run lambda:invoke_Setting_Group(self) but python expects lambda arg1:self.invoke_Setting_Group(self)
You could create function with extra argument
def invoke_My_method1(self, event):
print "expand another window"
print "event:", event, event.widget, event.x, event.y
And then you could use it
.bind("<<ComboboxSelected>>", lambda event:invoke_Setting_Group(self, event))
BTW: it looks strange - you have class App() but in second file you use only functions instead of some class too.