In ray, track.init has to be done before calling track.log. However, if i do track.log outside the function that defines the actor it gives me error complaining that no track session found. But if i do the following, i am creating a tracking session for every hyperparam value. Does the below sound right (doesnt look good to me since we are creating one tracking session per track.log)
def train(config, reporter=None):
track.init('track_session')
track.log(acc=acc)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--redis-address")
args = parser.parse_args()
ray.init(redis_address=args.redis_address)
config = {
"l1": ray.tune.grid_search([0.001, 0.01, 0.1]),
}
ray.tune.run(train, config=config)
Try this:
def train(config):
acc = 0
track.log(acc=acc)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--redis-address")
args = parser.parse_args()
ray.init(redis_address=args.redis_address)
config = {
"l1": ray.tune.grid_search([0.001, 0.01, 0.1]),
}
ray.tune.run(train, config=config)
Related
I have some tests for functions that use cache, for example:
Function:
#retry(stop=stop_after_attempt(3))
#cache.cached(timeout=60, key_prefix='resouce_group_list')
def get_azure_resource_groups():
data = []
resource_client = get_azure_resource_client()
for item in resource_client.resource_groups.list():
data.append(item)
return data
Test:
#patch("dev_maintenance.machines.get_azure_resource_client")
def test_get_azure_list_rg(get_azure_resource_client):
cache.clear()
data = []
with app.app_context():
ret = get_azure_resource_groups()
get_azure_resource_client.assert_called_once()
expected = get_azure_resource_client.return_value.resource_groups.list.return_value
assert len(get_azure_resource_client.return_value.method_calls) == 1
for item in expected:
data.append(item)
assert ret == data
cache.clear()
The above test works fine, it passes, no errors and the test is using cache.
But i got other tests, and the decorator here does not matter, it will give the same error if i change the decorator to #cache.cache:
Function:
#retry(stop=stop_after_attempt(3))
#cache.memoize(60)
def get_azure_machine_info(rg_name, machine_name, expand="instanceView"):
try:
compute_client = get_azure_compute_client()
return compute_client.virtual_machines.get(rg_name, machine_name, expand=expand)
except CloudError:
return None
Test:
#patch("dev_maintenance.machines.get_azure_compute_client")
def test_get_azure_machine_info (get_azure_compute_client):
cache.delete_memoized(get_azure_machine_info)
with app.app_context():
ret = get_azure_machine_info("rg1", "m1")
print(ret)
get_azure_compute_client.assert_called_once()
assert len(get_azure_compute_client.return_value.method_calls) == 1
assert (
ret == get_azure_compute_client.return_value.virtual_machines.get.return_value
)
get_azure_compute_client.return_value.virtual_machines.get.assert_called_once_with(
"rg1", "m1", expand="instanceView"
)
cache.delete_memoized(get_azure_machine_info)
Now here the test fails with the error on this line ret = get_azure_machine_info("rg1", "m1"):
value = None, from_value = PicklingError("Can't pickle <class 'unittest.mock.MagicMock'>: it's not the same object as unittest.mock.MagicMock")
> ???
E tenacity.RetryError: RetryError[<Future at 0x105c7c3d0 state=finished raised PicklingError>]
<string>:3: RetryError
I tried to mock the cache passing a patch decorator like:
#patch("dev_maintenance.machines.cache") or #patch("dev_maintenance.cache")
I tried to set the CACHE_TYPE to null in the test case, instantiating the cache object and passing the config:
cache = Cache()
cache.init_app(app, config={"CACHE_TYPE": "redis"})
but no success so far, any help?
This is a reference to an old answer, but I think that generally MagicMock objects aren't meant to be pickled: https://github.com/thadeusb/flask-cache/issues/52
That error message is different though, and this is more similar to what you are seeing:
Is there a way to make python pickle ignore "it's not the same object " errors
Maybe you could replace the domain prefix to the class like the answer above, but I am not sure it will overcome the other difficulties of pickling a MagicMock class:
`#patch("__main__.get_azure_compute_client")`
I'am writing the answer here for people that need to test functions that are cached with flask-caching and have the same error then me.
What i needed was to create an Object inside the test and make the mock_value = Object like this:
First i create a simple class:
class MachineInfo(object):
pass
Then in my test:
#patch("dev_maintenance.machines.get_azure_compute_client")
def test_get_azure_machine_info (get_azure_compute_client):
cache.clear()
expected_res = MachineInfo()
expected_res.id = "id"
expected_res.name = "machine1"
expected_res.location = "location"
expected_res.hardware_profile = "hardware"
expected_res.storage_profile = "storage"
expected_res.network_profile = "network_profile"
get_azure_compute_client.return_value.virtual_machines.get.return_value = expected_res
res = get_azure_machine_info("rg1", "m1")
assert res == expected_res
cache.clear()
Then i could assert function_call() == Object or function_call() == mock.return_value
This simulates what the actual azure returns, an object, so i just make the mock return the object that i created so i can simulate the function itself.
I am trying to send a file across the network using Twisted with the LineReceiver protocol. The issue I am seeing is that when I read a binary file and try to send the chunks they simply don't send.
I am reading the file using:
import json
import time
import threading
from twisted.internet import reactor, threads
from twisted.protocols.basic import LineReceiver
from twisted.internet import protocol
MaximumMsgSize = 15500
trySend = True
connectionToServer = None
class ClientInterfaceFactory(protocol.Factory):
def buildProtocol(self, addr):
return WoosterInterfaceProtocol(self._msgProcessor, self._logger)
class ClientInterfaceProtocol(LineReceiver):
def connectionMade(self):
connectionToServer = self
def _DecodeMessage(self, rawMsg):
header, body = json.loads(rawMsg)
return (header, json.loads(body))
def ProcessIncomingMsg(self, rawMsg, connObject):
# Decode raw message.
decodedMsg = self._DecodeMessage(rawMsg)
self.ProccessTransmitJobToNode(decodedMsg, connObject)
def _BuildMessage(self, id, msgBody = {}):
msgs = []
fullMsgBody = json.dumps(msgBody)
msgBodyLength = len(fullMsgBody)
totalParts = 1 if msgBodyLength <= MaximumMsgSize else \
int(math.ceil(msgBodyLength / MaximumMsgSize))
startPoint = 0
msgBodyPos = 0
for partNo in range(totalParts):
msgBodyPos = (partNo + 1) * MaximumMsgSize
header = {'ID' : id, 'MsgParts' : totalParts,
'MsgPart' : partNo }
msg = (header, fullMsgBody[startPoint:msgBodyPos])
jsonMsg = json.dumps(msg)
msgs.append(jsonMsg)
startPoint = msgBodyPos
return (msgs, '')
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
for filename in exportedFiles:
with open (filename, "rb") as exportFileHandle:
data = exportFileHandle.read().encode('base64')
params = {
ExportFileToMaster_Tag.TaskID : taskID,
ExportFileToMaster_Tag.FileContents : data,
ExportFileToMaster_Tag.Filename : filename
}
msgs, _ = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
connection.sendLine(m)
def lineReceived(self, data):
threads.deferToThread(self.ProcessIncomingMsg, data, self)
def ConnectFailed(reason):
print 'Connection failed..'
reactor.callLater(20, reactor.callFromThread, ConnectToServer)
def ConnectToServer():
print 'Connecting...'
from twisted.internet.endpoints import TCP4ClientEndpoint
endpoint = TCP4ClientEndpoint(reactor, 'localhost', 8181)
deferItem = endpoint.connect(factory)
deferItem.addErrback(ConnectFailed)
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
reactor.callFromThread(ConnectToServer)
factory = ClientInterfaceFactory()
protocol = ClientInterfaceProtocol()
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
Is there something I am doing wrong?file is sent, it's when the write is multi part or there are more than one file it struggles.
If a single write occurs then the m
Note: I have updated the question with a crude piece of sample code in the hope it makes sense.
_BuildMessage returns a two-tuple: (msgs, '').
Your network code iterates over this:
msgs = self._BuildMessage(MsgID.ExportFileToMaster, params)
for m in msgs:
So your network code first tries to send a list of json encoded data and then tries to send the empty string. It most likely raises an exception because you cannot send a list of anything using sendLine. If you aren't seeing the exception, you've forgotten to enable logging. You should always enable logging so you can see any exceptions that occur.
Also, you're using time.sleep and you shouldn't do this in a Twisted-based program. If you're doing this to try to avoid overloading the receiver, you should use TCP's native backpressure instead by registering a producer which can receive pause and resume notifications. Regardless, time.sleep (and your loop over all the data) will block the entire reactor thread and prevent any progress from being made. The consequence is that most of the data will be buffered locally before being sent.
Also, your code calls LineReceiver.sendLine from a non-reactor thread. This has undefined results but you can probably count on it to not work.
This loop runs in the main thread:
while 1:
time.sleep(0.01)
if connectionToServer == None: continue
if trySend == True:
protocol.ProccessTransmitJobToNode(None, None)
trySend = False
while the reactor runs in another thread:
netThread = threading.Thread(target=reactor.run, kwargs={"installSignalHandlers": False})
netThread.start()
ProcessTransmitJobToNode simply calls self.sendLine:
def ProccessTransmitJobToNode(self, msg, connection):
rootDir = '../documentation/configs/Wooster'
exportedFiles = ['consoleLog.txt', 'blob.dat']
params = {
'Status' : 'buildStatus',
'TaskID' : 'taskID',
'Name' : 'taskName',
'Exports' : len(exportedFiles),
}
msg, statusStr = self._BuildMessage(101, params)
connection.sendLine(msg[0])
You should probably remove the use of threading entirely from the application. Time-based events are better managed using reactor.callLater (your main-thread loop effectively generates a call to ProcessTransmitJobToNode once hundred times a second (modulo effects of the trySend flag)).
You may also want to take a look at https://github.com/twisted/tubes as a better way to manage large amounts of data with Twisted.
this is my code for in which I am trying to link two pages
from Tkinter import *
class Example(Frame):
def __init__( self , parent, controller ):
Frame.__init__(self, parent)
self.controller=controller
self.parent = parent
self.parent.title("f2")
self.parent.configure(background="royalblue4")
self.pack(fill=BOTH, expand=1)
w = 800
h = 500
sw = self.parent.winfo_screenwidth()
sh = self.parent.winfo_screenheight()
x = (sw - w)/2
y = (sh - h)/2
self.parent.geometry('%dx%d+%d+%d' % (w, h, x, y))
self.logbtn1 = Button(self,text="SIGN UP",font=("Copperplate Gothic Bold",16),bg=("dark green"),activebackground=("green"),command=lambda: controller.show_frame("D:\java prgms\minor\signup"))
self.logbtn1.place(x=325,y=175)
self.logbtn2 = Button(self, text="LOGIN",font=("Copperplate Gothic Bold",16),bg=("cyan"),activebackground=("yellow"),command=lambda: controller.show_frame("D:\java prgms\minor\log1"))
self.logbtn2.place(x=335,y=220)
self.pack()
def main():
root = Tk()
ex = Example(root,Frame)
root.mainloop()
if __name__ == '__main__':
main()
But here I am getting this error message:
AttributeError: class Frame has no attribute 'show_frame'
how to remove this error
First of all, you can not call lambda the way you did in command=lambda: controller.show_frame(...).
Suppose you did the necessary imports that I do not see in your current program, just replace those 2 statements (in 2 lines of your code) by : command=controller.show_frame(...).
Please read about how to use lambda
Second, your code contains an other error around this line:
if name == 'main':
main()
Change it to:
if __name__ == '__main__':
main()
After I fixed your error, I run your program successfully:
P.S. May be you would be interested in this post: What does if __name__ == "__main__": do?
I just ran into some strange behavior that has me stumped. I'm writing a simple little GUI for some in-house data processing. I want to allow a user to switch between a few different data-processing modes and input some parameters which define how the data is processed for each mode. The problem is that when the user inputs new parameters, the app ignores requests to switch modes.
The code below replicates the issue. I apologize for the size, this was the shortest code that replicates the problem.
import Tkinter as Tk
class foo(Tk.Frame):
def __init__(self):
self.master = master =Tk.Tk()
Tk.Frame.__init__(self,self.master) #Bootstrap
#Here mode and parameters as key, value pairs
self.data = {'a':'Yay',
'b':'Boo'
}
self.tex = Tk.Text(master=master)
self.tex.grid(row=0,column=0,rowspan=3,columnspan=4)
self.e = Tk.Entry(master=master)
self.e.grid(row=3,column=0,columnspan=4)
self.sv =Tk.StringVar()
self.sv.set('a')
self.b1 = Tk.Radiobutton(master=master,
text = 'a',
indicatoron = 0,
variable = self.sv,
value = 'a')
self.b2 = Tk.Radiobutton(master=master,
text = 'b',
indicatoron = 0,
variable = self.sv,
value = 'b')
self.b3 = Tk.Button(master = master,
text='Apply',command=self.Apply_Func)
self.b4 = Tk.Button(master = master,
text='Print',command=self.Print_Func)
self.b1.grid(row=4,column=0)
self.b2.grid(row=4,column=1)
self.b3.grid(row=4,column=2)
self.b4.grid(row=4,column=3)
def Apply_Func(self):
self.innerdata = self.e.get()
def Print_Func(self):
self.tex.insert(Tk.END,str(self.innerdata)+'\n')
#This is how I'm retrieving the user selected parameters
#property
def innerdata(self):
return self.data[self.sv.get()]
#And how I'm setting the user defined parameters
#innerdata.setter
def innerdata(self,value):
self.data[self.sv.get()] = value
if __name__ == "__main__":
app = foo()
app.mainloop()
Expected behavior:
1) Press button 'a' then 'print' prints:
Yay
2) Press button 'b' then 'print' prints:
Boo
3) Type 'Zep Rocks' into the entry field and press apply
4) Pressing 'print' now yields
Zep Rocks
5) Pressing 'a' then 'print' should yield
Yay
But instead yields
Zep Rocks
Which might be true, but not desired right now. What is going on here?
Edit: I have some new information. Tk.Frame in python 2.7 is not a new-style class. It isn't friendly with descriptors, so rather than interpreting the '=' as a request to use the foo.innerdata's __set__ method, it just assigns the result of self.e.get() to innerdata.
ARGLEBARGLE!!!
Still an open question: how do I get this to do what I want in a clean manner?
So the core problem is that Tk.Frame doesn't subclass from object, so it is not a new-style python class. Which means it doesn't get down with descriptors like I was trying to use. One solution that I found is to subclass my app from object instead.
Code that solves my problem is below:
import Tkinter as Tk
class foo(object):
def __init__(self,master):
self.master = master #Bootstrap
self.mainloop = master.mainloop
self.data = {'a':{'value':7,'metavalue':False},
'b':{'value':'Beeswax','metavalue':True}
}
self.tex = Tk.Text(master=master)
self.tex.grid(row=0,column=0,rowspan=3,columnspan=4)
self.e = Tk.Entry(master=master)
self.e.grid(row=3,column=0,columnspan=4)
self.sv =Tk.StringVar()
self.sv.set('a')
self.b1 = Tk.Radiobutton(master=master,
text = 'a',
indicatoron = 0,
variable = self.sv,
value = 'a')
self.b2 = Tk.Radiobutton(master=master,
text = 'b',
indicatoron = 0,
variable = self.sv,
value = 'b')
self.b3 = Tk.Button(master = master,text='Apply',command=self.Apply_Func)
self.b4 = Tk.Button(master = master,text='Print',command=self.Print_Func)
self.b1.grid(row=4,column=0)
self.b2.grid(row=4,column=1)
self.b3.grid(row=4,column=2)
self.b4.grid(row=4,column=3)
def Apply_Func(self):
self.innerdata = self.e.get()
def Print_Func(self):
self.tex.insert(Tk.END,str(self.innerdata)+'\n')
#property
def innerdata(self):
return self.data[self.sv.get()]
#innerdata.setter
def innerdata(self,value):
self.data[self.sv.get()] = value
if __name__ == "__main__":
master = Tk.Tk()
app = foo(master)
app.mainloop()
I am trying to invoke a method from combo box selected change event
with lambda expression but I am stuck with following error
TypeError: () takes no arguments (1 given)
I think I have passed 1 argument as per the method definition, could somebody please help me where I am wrong
or any other combobox selected change event code will be great help!
please note my code
self.boxWidget[boxName].bind("<<ComboboxSelected>>", lambda:invoke_Setting_Group(self))
def invoke_My_method1(self):
print "expand another window"
I am trying to pass the first class object to the second python script file for variable value assigning easeness.I tried to use this combox event change code without lambda then I noticed that this method is getting called automatically so I used lambda to prevent this automatic method calling
Sorry I am not having the knowledge on lambda expression usage; here I used only to prevent the automatic method execution. Without lambda expression I noticed my combo box function starts automatically, I did not understand why it happens so?
I am using TKinter python 2.6
More Detailed Code of above:
#Main_GUI_Class.py
##----------------------
import sys
class App():
def __init__ (self,master,geometry=None,root=None):
try:
self.master=master
if not root:
self.root=Tkinter.Toplevel(master)
def initUI(self):
try:
self.master.title("GUI")
menubar = Menu(self.master)
self.root.config(menu=menubar)
fileMenu.add_command(label='Open')
submenu_ncss.add_command(label='Model Setting',command=lambda:Combo_Expand_Script.Call_Model_Setting(self))
##----------------------
def main():
r = Tkinter.Tk()
r.withdraw()
r.title("GUI Sample")
r.wm_iconbitmap(Pic1)
v = App(r)
r.mainloop()
if __name__ == "__main__":
main()
##Combo_Expand_Script.py
##-----------------------
import sys
import Tkinter
import Main_GUI_Class
def Call_Model_Setting(self):
try:
self.PopUpWin = Toplevel(bg='#54596d',height=500, width=365)
self.PopUpWin.title("POP UP SETTING")
#Combo Boxs in Pop Up
boxNameGroup="boxSetting"
boxPlaceY=0
for Y in range(4):
boxName=boxNameGroup+str(Y)
if Y == 0:
boxPlaceY=50
else:
boxPlaceY=boxPlaceY+40
self.box_value = StringVar()
self.boxWidget[boxName] = ttk.Combobox(self.PopUpWin, height=1, width=20)
if Y== 0:
self.boxWidget[boxName]['values'] = ('A', 'B')
self.boxWidget[boxName].current(1)
if Y== 1:
self.boxWidget[boxName]['values'] = ('X', 'Y')
self.boxWidget[boxName].bind("<<ComboboxSelected>>",lambda:invoke_Setting_Group(self))
self.boxWidget[boxName].place(x=180, y = boxPlaceY)
#Buttons in Pop Up
self.btnApply = tk.Button(self.PopUpWin,width=10, height=1,text="Apply",relief=FLAT,bg=btn_Bg_Color,command=lambda: treeDataTransfer(self,0))
self.btnApply.pack()
self.btnApply.place(x=75, y = 460)
self.btnCancel = tk.Button(self.PopUpWin,width=10, height=1,text="Cancel",relief=FLAT,command=lambda: deleteTreeNodes(self))
self.btnCancel.pack()
self.btnCancel.place(x=170, y = 460)
except IOError:
print "Error: data error"
def invoke_Setting_Group(self):#, event=None
try:
#self.boxName.current(0)
self.boxWidget["boxSetting3"].current(0)
self.PopUpWin['width']=1050
self.PopUpWin['height']=700
self.btnApply.place(x=500, y = 550)
self.btnCancel.place(x=600, y = 550)
self.txtWidget={}
lsttxtSetting = ['1', '2','3 ','4','5 ','6','7','8','9','10']
for t in range(10):
txtName=txtNameGroupTS+str(t)
if t == 0:
txtPlaceY=120
else:
txtPlaceY=txtPlaceY+30
self.txtWidget[txtName] = Text(self.groupSettingFrame,height=1, width=10,borderwidth = 2)
self.txtWidget[txtName].insert(INSERT, lsttxtSetting[t])
self.txtWidget[txtName].pack()
self.txtWidget[txtName].place(x=200, y = txtPlaceY)
except IOError:
print "Error: Group Settings Popup error"
def turbDataTransferBind(self):
for P in range(0,3):
boxName="boxSetting"+str(X)
dataSettingbox=self.lstTurb[X]+" "+self.boxWidget[boxName].get()
self.root_node_Setting = self.tree.insert( self.root_node_ChildSetting["ChildSettingNode"], 'end', text=dataSettingbox, open=True)
def treeDataTransfer(self,dlgTurbFlag):
self.treeDataTransferBind()
print "data tranfer sucess"
def deleteTreeNodes(self):
print "delete nodes"
command= and bind expect function name - without () and arguments - so in place of
If you use
.bind("<<ComboboxSelected>>", invoke_Setting_Group(self) )
then you use result from invoke_Setting_Group(self) as second argument in .bind(). This way you could dynamicly generate function used as argument in bind
TypeError: () takes no arguments (1 given)
This means you have function function() but python run it as function(arg1)
You run lambda:invoke_Setting_Group(self) but python expects lambda arg1:self.invoke_Setting_Group(self)
You could create function with extra argument
def invoke_My_method1(self, event):
print "expand another window"
print "event:", event, event.widget, event.x, event.y
And then you could use it
.bind("<<ComboboxSelected>>", lambda event:invoke_Setting_Group(self, event))
BTW: it looks strange - you have class App() but in second file you use only functions instead of some class too.