Once a single function needs to be awaited, it seems like a cascading of functions that need to be awaited, until you reach a point where adding await breaks stuff:
I have dash code that intermittently gets called:
def get_data(symbolStr):
print("Getting AR for", symbolStr)
df = amp.get_data_job(symbolStr) #calls async code and need to be awaited
return df
...
html.Div(
dcc.Interval(id='interval1', interval = 900 * 1000, n_intervals=0),
),
html.Div(
dash_table.DataTable(
id='table',
columns=[{"name": i, "id": i} for i in magic_columns],
editable=True,
row_deletable=True
),
)
#app.callback(
[dash.dependencies.Output('table', 'data')],
[dash.dependencies.Input('interval1', 'n_intervals')],
[dash.dependencies.State('table', 'data')]
)
def update_interval(n, data):
allSymbolsDf = pd.DataFrame()
for symbolStr in symbolsStr:
df = get_data(symbolStr)
allSymbolsDf = allSymbolsDf.append(df)
time.sleep(3)
return [allSymbolsDf.to_dict('rows')]
Thing is, get_data is a function that calls asynchronous code
df = amp.get_data_job(symbolStr)
I can't turn the callback update_interval into an asynchronous function (AFAIK). What is the recommended way out of this. I mean, still keeping dash.
Ran into the same situation using FastApi and serving dash on the side. Use asyncio.run()
import asyncio
async def asyn_fun(foo):
return True if foo == "bar" else False
def sync_fun(foo):
return asyncio.run(asyn_fun(foo))
print(sync_fun("bar"))
Related
When I try to invoke the lambda function using a public url I don't see any output.
# Lambda function
import json
import hmac, base64, struct, hashlib, time
sec = 'my_secret'
def get_hotp_token(secret, intervals_no):
key = base64.b32decode(secret, True)
msg = struct.pack(">Q", intervals_no)
h = hmac.new(key, msg, hashlib.sha1).digest()
o = h[19] & 15
h = (struct.unpack(">I", h[o:o+4])[0] & 0x7fffffff) % 1000000
return h
def get_totp_token(secret):
return get_hotp_token(secret, intervals_no=int(time.time())//30)
def lambda_handler(event, totp):
# totp = get_totp_token(sec)
# print(totp)
return {
'statusCode' : 200,
'totp' : get_totp_token(sec)
}
When I test the function using aws-console I get the desired output.
I went to Configuration---->Function URL---->Create Function Url---->Auth type---->None----> Cors---->Yes. Saved with default settings of CORS.
Now when I use the URL I get a blank page.
Lamba body response should be a string, Not an object/ dict.
If you want to return a dict you'l need to stringify it.
https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html
I have some tests for functions that use cache, for example:
Function:
#retry(stop=stop_after_attempt(3))
#cache.cached(timeout=60, key_prefix='resouce_group_list')
def get_azure_resource_groups():
data = []
resource_client = get_azure_resource_client()
for item in resource_client.resource_groups.list():
data.append(item)
return data
Test:
#patch("dev_maintenance.machines.get_azure_resource_client")
def test_get_azure_list_rg(get_azure_resource_client):
cache.clear()
data = []
with app.app_context():
ret = get_azure_resource_groups()
get_azure_resource_client.assert_called_once()
expected = get_azure_resource_client.return_value.resource_groups.list.return_value
assert len(get_azure_resource_client.return_value.method_calls) == 1
for item in expected:
data.append(item)
assert ret == data
cache.clear()
The above test works fine, it passes, no errors and the test is using cache.
But i got other tests, and the decorator here does not matter, it will give the same error if i change the decorator to #cache.cache:
Function:
#retry(stop=stop_after_attempt(3))
#cache.memoize(60)
def get_azure_machine_info(rg_name, machine_name, expand="instanceView"):
try:
compute_client = get_azure_compute_client()
return compute_client.virtual_machines.get(rg_name, machine_name, expand=expand)
except CloudError:
return None
Test:
#patch("dev_maintenance.machines.get_azure_compute_client")
def test_get_azure_machine_info (get_azure_compute_client):
cache.delete_memoized(get_azure_machine_info)
with app.app_context():
ret = get_azure_machine_info("rg1", "m1")
print(ret)
get_azure_compute_client.assert_called_once()
assert len(get_azure_compute_client.return_value.method_calls) == 1
assert (
ret == get_azure_compute_client.return_value.virtual_machines.get.return_value
)
get_azure_compute_client.return_value.virtual_machines.get.assert_called_once_with(
"rg1", "m1", expand="instanceView"
)
cache.delete_memoized(get_azure_machine_info)
Now here the test fails with the error on this line ret = get_azure_machine_info("rg1", "m1"):
value = None, from_value = PicklingError("Can't pickle <class 'unittest.mock.MagicMock'>: it's not the same object as unittest.mock.MagicMock")
> ???
E tenacity.RetryError: RetryError[<Future at 0x105c7c3d0 state=finished raised PicklingError>]
<string>:3: RetryError
I tried to mock the cache passing a patch decorator like:
#patch("dev_maintenance.machines.cache") or #patch("dev_maintenance.cache")
I tried to set the CACHE_TYPE to null in the test case, instantiating the cache object and passing the config:
cache = Cache()
cache.init_app(app, config={"CACHE_TYPE": "redis"})
but no success so far, any help?
This is a reference to an old answer, but I think that generally MagicMock objects aren't meant to be pickled: https://github.com/thadeusb/flask-cache/issues/52
That error message is different though, and this is more similar to what you are seeing:
Is there a way to make python pickle ignore "it's not the same object " errors
Maybe you could replace the domain prefix to the class like the answer above, but I am not sure it will overcome the other difficulties of pickling a MagicMock class:
`#patch("__main__.get_azure_compute_client")`
I'am writing the answer here for people that need to test functions that are cached with flask-caching and have the same error then me.
What i needed was to create an Object inside the test and make the mock_value = Object like this:
First i create a simple class:
class MachineInfo(object):
pass
Then in my test:
#patch("dev_maintenance.machines.get_azure_compute_client")
def test_get_azure_machine_info (get_azure_compute_client):
cache.clear()
expected_res = MachineInfo()
expected_res.id = "id"
expected_res.name = "machine1"
expected_res.location = "location"
expected_res.hardware_profile = "hardware"
expected_res.storage_profile = "storage"
expected_res.network_profile = "network_profile"
get_azure_compute_client.return_value.virtual_machines.get.return_value = expected_res
res = get_azure_machine_info("rg1", "m1")
assert res == expected_res
cache.clear()
Then i could assert function_call() == Object or function_call() == mock.return_value
This simulates what the actual azure returns, an object, so i just make the mock return the object that i created so i can simulate the function itself.
I am trying to send a file across the network using Twisted with the LineReceiver protocol. The issue I am seeing is that when I read a binary file and try to send the chunks they simply don't send.
I am reading the file using:
import json
import time
import threading
from twisted.internet import reactor, threads
from twisted.protocols.basic import LineReceiver
from twisted.internet import protocol
MaximumMsgSize = 15500
trySend = True
connectionToServer = None
class ClientInterfaceFactory(protocol.Factory):
def buildProtocol(self, addr):
return WoosterInterfaceProtocol(self._msgProcessor, self._logger)
class ClientInterfaceProtocol(LineReceiver):
def connectionMade(self):
connectionToServer = self
def _DecodeMessage(self, rawMsg):
header, body = json.loads(rawMsg)
return (header, json.loads(body))
def ProcessIncomingMsg(self, rawMsg, connObject):
# Decode raw message.
decodedMsg = self._DecodeMessage(rawMsg)
self.ProccessTransmitJobToNode(decodedMsg, connObject)
def _BuildMessage(self, id, msgBody = {}):
msgs = []
fullMsgBody = json.dumps(msgBody)
msgBodyLength = len(fullMsgBody)
totalParts = 1 if msgBodyLength <= MaximumMsgSize else \
int(math.ceil(msgBodyLength / MaximumMsgSize))
startPoint = 0
msgBodyPos = 0
for partNo in range(totalParts):
msgBodyPos = (partNo + 1) * MaximumMsgSize
header = {'ID' : id, 'MsgParts' : totalParts,
'MsgPart' : partNo }
msg = (header, fullMsgBody[startPoint:msgBodyPos])
jsonMsg = json.dumps(msg)
msgs.append(jsonMsg)
startPoint = msgBodyPos
return (msgs, '')
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
for filename in exportedFiles:
with open (filename, "rb") as exportFileHandle:
data = exportFileHandle.read().encode('base64')
params = {
ExportFileToMaster_Tag.TaskID : taskID,
ExportFileToMaster_Tag.FileContents : data,
ExportFileToMaster_Tag.Filename : filename
}
msgs, _ = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
connection.sendLine(m)
def lineReceived(self, data):
threads.deferToThread(self.ProcessIncomingMsg, data, self)
def ConnectFailed(reason):
print 'Connection failed..'
reactor.callLater(20, reactor.callFromThread, ConnectToServer)
def ConnectToServer():
print 'Connecting...'
from twisted.internet.endpoints import TCP4ClientEndpoint
endpoint = TCP4ClientEndpoint(reactor, 'localhost', 8181)
deferItem = endpoint.connect(factory)
deferItem.addErrback(ConnectFailed)
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
reactor.callFromThread(ConnectToServer)
factory = ClientInterfaceFactory()
protocol = ClientInterfaceProtocol()
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
Is there something I am doing wrong?file is sent, it's when the write is multi part or there are more than one file it struggles.
If a single write occurs then the m
Note: I have updated the question with a crude piece of sample code in the hope it makes sense.
_BuildMessage returns a two-tuple: (msgs, '').
Your network code iterates over this:
msgs = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
So your network code first tries to send a list of json encoded data and then tries to send the empty string. It most likely raises an exception because you cannot send a list of anything using sendLine. If you aren't seeing the exception, you've forgotten to enable logging. You should always enable logging so you can see any exceptions that occur.
Also, you're using time.sleep and you shouldn't do this in a Twisted-based program. If you're doing this to try to avoid overloading the receiver, you should use TCP's native backpressure instead by registering a producer which can receive pause and resume notifications. Regardless, time.sleep (and your loop over all the data) will block the entire reactor thread and prevent any progress from being made. The consequence is that most of the data will be buffered locally before being sent.
Also, your code calls LineReceiver.sendLine from a non-reactor thread. This has undefined results but you can probably count on it to not work.
This loop runs in the main thread:
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
while the reactor runs in another thread:
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
ProcessTransmitJobToNode simply calls self.sendLine:
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
You should probably remove the use of threading entirely from the application. Time-based events are better managed using reactor.callLater (your main-thread loop effectively generates a call to ProcessTransmitJobToNode once hundred times a second (modulo effects of the trySend flag)).
You may also want to take a look at https://github.com/twisted/tubes as a better way to manage large amounts of data with Twisted.
Please advise how can I keep order while using multiprocessing in Python. Both func and somefunc return lists. Thanks,
results = []
def callback(*args): results.append(*args)
def in_parallel(fn, func):
print 'Process started on',time.strftime('%H:%M:%S')
pool = mp.Pool(processes = 4)
for num, i in enumerate(somefunc(fn)):
pool.apply_async(func, args = (i,), callback = callback([num, callback]))
pool.close()
pool.join()
return results
The problem has been solved with the following:
r = pool.map_async(...)
r.wait()
I am trying to invoke a method from combo box selected change event
with lambda expression but I am stuck with following error
TypeError: () takes no arguments (1 given)
I think I have passed 1 argument as per the method definition, could somebody please help me where I am wrong
or any other combobox selected change event code will be great help!
please note my code
self.boxWidget[boxName].bind("<<ComboboxSelected>>", lambda:invoke_Setting_Group(self))
def invoke_My_method1(self):
print "expand another window"
I am trying to pass the first class object to the second python script file for variable value assigning easeness.I tried to use this combox event change code without lambda then I noticed that this method is getting called automatically so I used lambda to prevent this automatic method calling
Sorry I am not having the knowledge on lambda expression usage; here I used only to prevent the automatic method execution. Without lambda expression I noticed my combo box function starts automatically, I did not understand why it happens so?
I am using TKinter python 2.6
More Detailed Code of above:
#Main_GUI_Class.py
##----------------------
import sys
class App():
def __init__ (self,master,geometry=None,root=None):
try:
self.master=master
if not root:
self.root=Tkinter.Toplevel(master)
def initUI(self):
try:
self.master.title("GUI")
menubar = Menu(self.master)
self.root.config(menu=menubar)
fileMenu.add_command(label='Open')
submenu_ncss.add_command(label='Model Setting',command=lambda:Combo_Expand_Script.Call_Model_Setting(self))
##----------------------
def main():
r = Tkinter.Tk()
r.withdraw()
r.title("GUI Sample")
r.wm_iconbitmap(Pic1)
v = App(r)
r.mainloop()
if __name__ == "__main__":
main()
##Combo_Expand_Script.py
##-----------------------
import sys
import Tkinter
import Main_GUI_Class
def Call_Model_Setting(self):
try:
self.PopUpWin = Toplevel(bg='#54596d',height=500, width=365)
self.PopUpWin.title("POP UP SETTING")
#Combo Boxs in Pop Up
boxNameGroup="boxSetting"
boxPlaceY=0
for Y in range(4):
boxName=boxNameGroup+str(Y)
if Y == 0:
boxPlaceY=50
else:
boxPlaceY=boxPlaceY+40
self.box_value = StringVar()
self.boxWidget[boxName] = ttk.Combobox(self.PopUpWin, height=1, width=20)
if Y== 0:
self.boxWidget[boxName]['values'] = ('A', 'B')
self.boxWidget[boxName].current(1)
if Y== 1:
self.boxWidget[boxName]['values'] = ('X', 'Y')
self.boxWidget[boxName].bind("<<ComboboxSelected>>",lambda:invoke_Setting_Group(self))
self.boxWidget[boxName].place(x=180, y = boxPlaceY)
#Buttons in Pop Up
self.btnApply = tk.Button(self.PopUpWin,width=10, height=1,text="Apply",relief=FLAT,bg=btn_Bg_Color,command=lambda: treeDataTransfer(self,0))
self.btnApply.pack()
self.btnApply.place(x=75, y = 460)
self.btnCancel = tk.Button(self.PopUpWin,width=10, height=1,text="Cancel",relief=FLAT,command=lambda: deleteTreeNodes(self))
self.btnCancel.pack()
self.btnCancel.place(x=170, y = 460)
except IOError:
print "Error: data error"
def invoke_Setting_Group(self):#, event=None
try:
#self.boxName.current(0)
self.boxWidget["boxSetting3"].current(0)
self.PopUpWin['width']=1050
self.PopUpWin['height']=700
self.btnApply.place(x=500, y = 550)
self.btnCancel.place(x=600, y = 550)
self.txtWidget={}
lsttxtSetting = ['1', '2','3 ','4','5 ','6','7','8','9','10']
for t in range(10):
txtName=txtNameGroupTS+str(t)
if t == 0:
txtPlaceY=120
else:
txtPlaceY=txtPlaceY+30
self.txtWidget[txtName] = Text(self.groupSettingFrame,height=1, width=10,borderwidth = 2)
self.txtWidget[txtName].insert(INSERT, lsttxtSetting[t])
self.txtWidget[txtName].pack()
self.txtWidget[txtName].place(x=200, y = txtPlaceY)
except IOError:
print "Error: Group Settings Popup error"
def turbDataTransferBind(self):
for P in range(0,3):
boxName="boxSetting"+str(X)
dataSettingbox=self.lstTurb[X]+" "+self.boxWidget[boxName].get()
self.root_node_Setting = self.tree.insert( self.root_node_ChildSetting["ChildSettingNode"], 'end', text=dataSettingbox, open=True)
def treeDataTransfer(self,dlgTurbFlag):
self.treeDataTransferBind()
print "data tranfer sucess"
def deleteTreeNodes(self):
print "delete nodes"
command= and bind expect function name - without () and arguments - so in place of
If you use
.bind("<<ComboboxSelected>>", invoke_Setting_Group(self) )
then you use result from invoke_Setting_Group(self) as second argument in .bind(). This way you could dynamicly generate function used as argument in bind
TypeError: () takes no arguments (1 given)
This means you have function function() but python run it as function(arg1)
You run lambda:invoke_Setting_Group(self) but python expects lambda arg1:self.invoke_Setting_Group(self)
You could create function with extra argument
def invoke_My_method1(self, event):
print "expand another window"
print "event:", event, event.widget, event.x, event.y
And then you could use it
.bind("<<ComboboxSelected>>", lambda event:invoke_Setting_Group(self, event))
BTW: it looks strange - you have class App() but in second file you use only functions instead of some class too.