I am working on a parallelized optimization problem in PyGMO. Unfortunately the docs are not very helpful. According to these guidelines, I am riting my problem as
import PyGMO as pygmo
class my_problem(pygmo.base):
def __init__(self,model,config,pars,**kwargs):
# Does some parameter definition according to input arguments model, config etc...
...
# Invoke base class as required by PyGMO
super(my_problem,self).__init__(self.__dim)
def _objfun_impl(self,x):
...
f = ... # Cost function to optimize
return (f,)
# Main
model = 'ei'
config = 'x1'
args = (...)
prob = my_problem(model,config,args)
algo = pygmo.algorithm.de(gen=20)
isl = pygmo.island(algo,prob,20)
print isl.population.champion.f
isl.evolve(10)
print isl.population.champion.f
This does not work and produce the following error:
File "/home/maurizio/Dropbox/Stability_Analysis_network/mymain.py", line 643, in main_routine
isl = pygmo.island(algo,prob,20)
File "/usr/lib/python2.7/site-packages/PyGMO/core/__init__.py", line 239, in island
return _generic_island_ctor(None, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/PyGMO/core/__init__.py", line 132, in _generic_island_ctor
return py_island(*args, **kwargs)
File "/usr/lib/python2.7/site-packages/PyGMO/core/__init__.py", line 119, in _generic_island_ctor
super(type(self), self).__init__(*ctor_args)
File "/usr/lib/python2.7/site-packages/PyGMO/core/__init__.py", line 48, in __init__
_core._base_island.__init__(self, *args)
File "/usr/lib/python2.7/site-packages/PyGMO/problem/_base.py", line 36, in __get_deepcopy__
return deepcopy(self)
File "/usr/lib64/python2.7/copy.py", line 190, in deepcopy
y = _reconstruct(x, rv, 1, memo)
File "/usr/lib64/python2.7/copy.py", line 329, in _reconstruct
y = callable(*args)
TypeError: __init__() takes exactly 4 arguments (1 given)
Any idea to what __init__ is referring to and what parameters are missing? I suspect that it is a problem with my class definition.
The issue is caused by the mismatch between input arguments of my_problem.__init__(...) (i.e. the Child class) and the base.__init__ (i.e. the Parent class). If defaults for these parameters are not provided then a conflict results by the inheritance of __init__ by super(my_problem,self) from base. In practice, a corrected working version is:
import PyGMO as pygmo
class my_problem(pygmo.base):
def __init__(self,model='ei',config='conf1',pars=[1]*20):
# Does some parameter definition according to input arguments model, config etc...
...
self.__dim = 3
...
# Invoke base class as required by PyGMO
super(my_problem,self).__init__(self.__dim)
def _objfun_impl(self,x):
...
f = ... # Cost function to optimize
return (f,)
# Main
...
It is not possible instead to pass **kwargs to the child class insofar as base is hard coded and should be changed accordingly to this post.
Related
I am writing a script for pulp Optimisation to engage with my Django database. The problem contains a few thousand variables to be optimised and several hundred constraints which vary depending upon the values of a,b,c.
var_names[]
var_values{}
for _foo_ in list_1:
for _bar_ in list_2:
for _var_ in list_3:
for _eet_ list_4:
var_name = str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)
var_names.append(var_name)
exec(str(_foo_)+str(_bar_)+str(_var_)+str(_eet_) + "= LpVariable("str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)+", lowBound=0, cat='Integer')")
var_value = DataBase.objects.get(column_A = str(_foo_)+str(_var_)).value
var_values.append(var_value)
obj_func = LpAffineExpression([var_names[i],var_values[i] for in in range(len(var_names))])
problem = LpProblem(name="name", sense=LpMinimise)
#Example of constraints
exec("problem += (" + str(function(a1,b1,c1)_) +str(function(a1,b1,c2)) +" >= Database_2.objects.get(column_A = z1).value")
problem += obj_func
problem.sovle()
The code works in jupyter notebook when I load the database info as a dataframe. However, I keep receiving this following error code when using in Djagno:
File "/path/to/files/prob.py", line 1610, in <module>
problem.solve()
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1913, in solve
status = solver.actualSolve(self, **kwargs)
File "/path/to/files/lib/python3.9/site-packages/pulp/apis/coin_api.py", line 137, in actualSolve
return self.solve_CBC(lp, **kwargs)
File "/path/to/files/lib/python3.9/site-packages/pulp/apis/coin_api.py", line 153, in solve_CBC
vs, variablesNames, constraintsNames, objectiveName = lp.writeMPS(
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1782, in writeMPS
return mpslp.writeMPS(self, filename, mpsSense=mpsSense, rename=rename, mip=mip)
File "/path/to/files/lib/python3.9/site-packages/pulp/mps_lp.py", line 204, in writeMPS
constrNames, varNames, cobj.name = LpProblem.normalisedNames()
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1546, in normalisedNames
_variables = self.variables()
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1624, in variables
self.addVariables(list(self.objective.keys()))
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1614, in addVariables
self.addVariable(v)
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1603, in addVariable
if variable.hash not in self._variable_ids:
AttributeError: 'str' object has no attribute 'hash'
I have the code stored as .py file which is called views.py.
I believe it may be an issue with the namespace of the LpVariable creations. I have tried to:
define the whole problem encapsulated as a function with no entry variable and returns a dict of the solution.
Define the problem as a class with problem.create and problem.solve as methods to create the variables and solve the function.
Update the exec() code to store variables in the globals diction.
exec(str(_foo_)+str(_bar_)+str(_var_)+str(_eet_) + "= LpVariable("str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)+", lowBound=0, cat='Integer')", globals())
And alternately creating a local dict and executing the above code with locals(),local_dict .
Used LpVariable.dict
variables = LpVariable.dicts("variables", [(_foo_, _bar_, _var_, _eet_), for _foo_ in list1 for _bar_ in list2 for _var_ in list3 for _eet_ in list4], lowBound=o, cat="Integer")
This does create all the variables, however the function used in the constraints references the variables as per the name str(foo)+str(bar)+str(var)+str(eet) and not variable[i] , which then generates undefined variable errors.
As mentioned, this code does work in jupyter, I am just at a loss as to what the error may be a result of.
LpAffineExpression expects LpVariable, not var_names[i].
var_items = []
# exec(str(_foo_)+str(_bar_)+str(_var_)+str(_eet_) + "= LpVariable(" + str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)+", lowBound=0, cat='Integer')")
# var_value = DataBase.objects.get(column_A = str(_foo_)+str(_var_)).value
# var_values.append(var_value)
var_value = LpVariable(var_name, lowBound=0, cat='Integer')
var_coeff = DataBase.objects.get(column_A = str(_foo_)+str(_var_)).value
var_items.append((var_value, var_coeff))
# obj_func = LpAffineExpression([var_names[i],var_values[i] for i in range(len(var_names))])
obj_func = LpAffineExpression(var_items)
Reference: https://coin-or.github.io/pulp/technical/pulp.html#pulp.LpAffineExpression
I'm trying to run a minimization problem using scipy.optimize, including a NonlinearConstraint. I really don't want to code derivatives myself, so I'm using autograd to do it. But even though I follow the exact same procedure for the arguments to minimize and to NonlinearConstraint, the first seems to work and the second doesn't.
Here's my MWE:
useconstraint = False
import autograd
import autograd.numpy as np
from scipy import optimize
def function(x): return x[0]**2 + x[1]**2
functionjacobian = autograd.jacobian(function)
functionhvp = autograd.hessian_vector_product(function)
def constraint(x): return np.array([x[0]**2 - x[1]**2])
constraintjacobian = autograd.jacobian(constraint)
constrainthvp = autograd.hessian_vector_product(constraint)
constraint = optimize.NonlinearConstraint(constraint, 1, np.inf, constraintjacobian, constrainthvp)
startpoint = [1, 2]
bounds = optimize.Bounds([-np.inf, -np.inf], [np.inf, np.inf])
print optimize.minimize(
function,
startpoint,
method='trust-constr',
jac=functionjacobian,
hessp=functionhvp,
constraints=[constraint] if useconstraint else [],
bounds=bounds,
)
When I turn useconstraint off (at the top), it works fine and minimizes at (0, 0) as expected. When I turn it on, I get the following error:
Traceback (most recent call last):
File "test.py", line 29, in <module>
bounds=bounds,
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_minimize.py", line 613, in minimize
callback=callback, **options)
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_trustregion_constr/minimize_trustregion_constr.py", line 336, in _minimize_trustregion_constr
for c in constraints]
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_constraints.py", line 213, in __init__
finite_diff_bounds, sparse_jacobian)
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_differentiable_functions.py", line 343, in __init__
self.H = hess(self.x, self.v)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/wrap_util.py", line 20, in nary_f
return unary_operator(unary_f, x, *nary_op_args, **nary_op_kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/differential_operators.py", line 24, in grad
vjp, ans = _make_vjp(fun, x)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/core.py", line 10, in make_vjp
end_value, end_node = trace(start_node, fun, x)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/tracer.py", line 10, in trace
end_box = fun(start_box)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/wrap_util.py", line 15, in unary_f
return fun(*subargs, **kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/differential_operators.py", line 88, in vector_dot_grad
return np.tensordot(fun_grad(*args, **kwargs), vector, np.ndim(vector))
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/tracer.py", line 44, in f_wrapped
ans = f_wrapped(*argvals, **kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/tracer.py", line 48, in f_wrapped
return f_raw(*args, **kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/numpy/core/numeric.py", line 1371, in tensordot
raise ValueError("shape-mismatch for sum")
ValueError: shape-mismatch for sum
What am I doing wrong? I think the issue is in the hessian_vector_product because I see hess in the error message, but I'm not sure about that.
Ok, I found the answer. This was very confusing.
The hessp argument to minimize expects a function that returns the "Hessian of objective function times an arbitrary vector p" (source). By contrast, the hess argument to NonlinearConstraint expects "A callable [that] must return the Hessian matrix of dot(fun, v)" (source).
If you interpret the first quote like I did, "Hessian of (objective function times an arbitrary vector p)", it means pretty much the same thing as "Hessian matrix of dot(fun, v)". I therefore assumed you could use the same autograd function for both.
However, the correct interpretation is "(Hessian of objective function) times an arbitrary vector p", which is completely different. The hessian_vector_product function in autograd gives the correct result for the first, but you need a different function for the second.
thisfile.py
import cPickle
import gzip
import os
import numpy
import theano
import theano.tensor as T
def load_data(dataset):
f = gzip.open(dataset, 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
def shared_dataset(data_xy, borrow=True):
data_x, data_y = data_xy
shared_x = theano.shared(numpy.asarray(data_x,
dtype=theano.config.floatX),
borrow=borrow)
shared_y = theano.shared(numpy.asarray(data_y,
dtype=theano.config.floatX),
borrow=borrow)
return shared_x, T.cast(shared_y, 'int32')
test_set_x, test_set_y = shared_dataset(test_set)
valid_set_x, valid_set_y = shared_dataset(valid_set)
train_set_x, train_set_y = shared_dataset(train_set)
rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),
(test_set_x, test_set_y)]
return rval
class PCA(object):
def __init__(self):
self.param = 0
def dimemsion_transform(self, X):
m_mean = T.mean(X, axis=0)
X = X - m_mean ##################### this line makes error
return X
if __name__ == '__main__':
dataset = 'mnist.pkl.gz'
# load the MNIST data
data = load_data(dataset)
X = T.matrix('X')
m_pca = PCA()
transform = theano.function(
inputs=[],
outputs=m_pca.dimemsion_transform(X),
givens={
X: data
}
)
error showing like below
Traceback (most recent call last):
File ".../thisfile.py", line 101, in <module>
X: data
File ".../Theano/theano/compile/function.py", line 322, in function
output_keys=output_keys)
File ".../Theano/theano/compile/pfunc.py", line 443, in pfunc
no_default_updates=no_default_updates)
File ".../Theano/theano/compile/pfunc.py", line 219, in rebuild_collect_shared
cloned_v = clone_v_get_shared_updates(v, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 93, in clone_v_get_shared_updates
clone_v_get_shared_updates(i, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 93, in clone_v_get_shared_updates
clone_v_get_shared_updates(i, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 93, in clone_v_get_shared_updates
clone_v_get_shared_updates(i, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 96, in clone_v_get_shared_updates
[clone_d[i] for i in owner.inputs], strict=rebuild_strict)
File ".../Theano/theano/gof/graph.py", line 242, in clone_with_new_inputs
new_inputs[i] = curr.type.filter_variable(new)
File ".../Theano/theano/tensor/type.py", line 234, in filter_variable
self=self))
TypeError: Cannot convert Type Generic (of Variable <Generic>) into Type TensorType(float64, matrix). You can try to manually convert <Generic> into a TensorType(float64, matrix).
I am making PCA function with theano but have a problem.
mean value is subtracted from MNIST data in dimension_transform in PCA class
I do not get why it gives type matching error and how do I fix it
Your problem comes from these lines:
data = load_data(dataset)
Here data is a list (as this is what load_data() returns).
transform = theano.function(
inputs=[],
outputs=m_pca.dimemsion_transform(X),
givens={
X: data
}
)
And here you pass it as a value. You have to extract the item you want from the return value of load_data() like so:
[(train_set_x, train_set_y), (valid_set_x, valid_set_y),
(test_set_x, test_set_y)] = load_data(dataset)
and then use
givens={
X: train_set_x
}
or one of the other values.
The code:
class StockFactory(UniqueObjectsFactory):
FACTORY_FOR = Stock
FACTORY_DJANGO_GET_OR_CREATE = ('name', 'market')
market = factory.SubFactory(MarketFactory)
symbol = FuzzyAttribute(lambda: ''.join(random.choice(string.ascii_uppercase) for _ in xrange(4)))
name = FuzzyCompanyName()
# last_trade_price = fuzzy.FuzzyDecimal(0.0, 10000.0)
class PositionsFactory(FreezeableFactory):
FACTORY_FOR = Position
FACTORY_DJANGO_GET_OR_CREATE = ('stock','AnotherObject')
id = FuzzyInteger(100000)
stock = factory.SubFactory(Stock)
AnotherObject = factory.SubFactory(AnotherObject) #If I comment stock out it would fail here
created_date = FuzzyDateTime(start_dt=datetime(2013, 1, 1, tzinfo=compat.UTC))
The error:
File "/home/alon/Projects/stox-server/execution/tests/functional/test_positions.py", line 21, in setUp
PositionsFactory.create( portfolio=self.portfolio)
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/base.py", line 522, in create
attrs = cls.attributes(create=True, extra=kwargs)
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/base.py", line 365, in attributes
force_sequence=force_sequence,
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/containers.py", line 283, in build
return stub.__fill__()
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/containers.py", line 83, in __fill__
res[attr] = getattr(self, attr)
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/containers.py", line 105, in __getattr__
val = val.evaluate(self, self.__containers)
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/containers.py", line 215, in evaluate
containers=containers,
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/declarations.py", line 75, in evaluate
return self.function(obj)
File "/home/alon/Projects/stox-server/execution/tests/common/factories.py", line 173, in <lambda>
symbol = factory.LazyAttribute(lambda pos: pos.stock.symbol)
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/containers.py", line 105, in __getattr__
val = val.evaluate(self, self.__containers)
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/containers.py", line 215, in evaluate
containers=containers,
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/declarations.py", line 299, in evaluate
return self.generate(sequence, obj, create, defaults)
File "/home/alon/.virtualenvs/stox-server/local/lib/python2.7/site-packages/factory/declarations.py", line 386, in generate
return subfactory.simple_generate(create, **params)
AttributeError: type object 'Stock' has no attribute 'simple_generate'
Any clues? Ideas? I work with factory-boy quite a lot and most of the time it's an excellent tool. but after hours of debugging I just cant find the problem
So stupid of me:
Those line are wrong:
stock = factory.SubFactory(Stock)
AnotherObject = factory.SubFactory(AnotherObject)
Should have been:
stock = factory.SubFactory(StockFactory)
AnotherObject = factory.SubFactory(AnotherObjectFactory)
Hope it helps anyone else who bumps into this issue
I'm trying to get a "getting started with pickles" script working. I managed to save a pickle file from a file, and load it. But when I save a pickle file in one file (the main.py in this case) and load it from another, I get an error. I probably missed something small, but can't figure out what.
main.py
import pickle
class Node:
"""This class represents a node"""
def __init__(self, value = None):
self.val = value
def toString(self):
return self.val
class Link:
"""This class represents a link between 2 nodes"""
def __init__(self, sourceNode, targetNode, LinkWigth):
self.source = sourceNode
self.target = targetNode
self.wight = LinkWigth
def setWeight(self, newWeight):
self.wight = newWeight
def toString(self):
return self.wight
class Graph:
"""This class represents a graph"""
def __init__(self):
self.nodes = []
self.links = []
def addNode(self, node):
self.nodes.append(node)
def addLink(self, link):
self.links.append(link)
def getInDegree(self, node):
counter = 0
for link in self.links:
if link.target == node:
counter +=1
else:
print "target is: %s" % link.target.toString()
print "source is: %s" % link.source.toString()
return counter
def toString(self):
for link in self.links:
print link.toString()
for node in self.nodes:
print node.toString()
if __name__ == "__main__":
n1 = Node(4)
l1 = Link(n1, n1, 1)
g = Graph()
g.addNode(n1)
g.addLink(l1)
pickle.dump(g, open('haha', 'wb') )
pickleLoader.py
import pickle
import main
n = main.Node(44)
print n.toString()
g = pickle.load( open('haha', 'rb') )
print "ha"
The error
C:\Users\R\Desktop\pickle test>main.py
C:\Users\R\Desktop\pickle test>pickleLoader.py
44
Traceback (most recent call last):
File "C:\Users\R\Desktop\pickle test\pickleLoader.py", line 7, in <module>
g = pickle.load( open('haha', 'rb') )
File "C:\Program Files\Python27\lib\pickle.py", line 1378, in load
return Unpickler(file).load()
File "C:\Program Files\Python27\lib\pickle.py", line 858, in load
dispatch[key](self)
File "C:\Program Files\Python27\lib\pickle.py", line 1069, in load_inst
klass = self.find_class(module, name)
File "C:\Program Files\Python27\lib\pickle.py", line 1126, in find_class
klass = getattr(mod, name)
AttributeError: 'module' object has no attribute 'Graph'
C:\Users\R\Desktop\pickle test>
I guess that the problem is something with the namespace because main.py has been imported, but I have no idea how to get it working.
This does appear to be related to how the classes are defined in relation to the module. A quick way to allow this to work is to import the components of the main module directly into pickleLoader:
from main import Graph, Node, Link
A better solution might be to move the common components (Graph, Node, Link) into their own module, and then import that module into both main and pickleLoader.