I wrote a script to do interpolation
import scipy.interpolate
import csv
inputfile1 = 'test.csv'
outputfile = 'Day1_out.csv'
distance_list = []
EC_list = []
new_dist_list=[]
outfile = open(outputfile,'w')
outfile.write('Distance,EC\n')
with open (inputfile1,'rb') as csvfile:
f1 = csv.reader(csvfile,delimiter=',')
next(f1) #skip header line
for row in f1:
dist = row[12]
EC=row[13]
distance_list.append(dist)
EC_list.append(EC)
y_interp = scipy.interpolate.interp1d(distance_list,EC_list)
new_dist = 561.7
end = 560.2
while new_dist>end:
new_dist_list.append(dist)
new_dist=new_dist-0.2
for distance in new_dist_list:
EC=y_interp(distance)
outfile.write(str(distance)+','+str(EC)+'\n')
outfile.close()
When I ran the script it gave me the error message
Traceback (most recent call last):
File "D:\14046\Scripts\interpolation_RoR.py", line 41, in <module>
EC=y_interp(distance)
File "C:\Python27\lib\site-packages\scipy\interpolate\polyint.py", line 54, in __call__
y = self._evaluate(x)
File "C:\Python27\lib\site-packages\scipy\interpolate\interpolate.py", line 448, in _evaluate
out_of_bounds = self._check_bounds(x_new)
File "C:\Python27\lib\site-packages\scipy\interpolate\interpolate.py", line 474, in _check_bounds
if self.bounds_error and below_bounds.any():
AttributeError: 'bool' object has no attribute 'any'
Anyone has any idea where I have errors?
BTW, the input file have these values for distance and EC
Distance,EC
561.8,450
561.78,446
561.7,444
561.2,440
561.02,438
560.5,437
560.1,435
Thanks,
We are getting the same error message here. I think this does not necessarily need to be a problem with your code.
In our case switching to SciPy version 0.15.0 instead of 0.13.x solves the problem.
So it looks like the current version of SciPy accepts a wider range of input values.
Related
I am writing a script for pulp Optimisation to engage with my Django database. The problem contains a few thousand variables to be optimised and several hundred constraints which vary depending upon the values of a,b,c.
var_names[]
var_values{}
for _foo_ in list_1:
for _bar_ in list_2:
for _var_ in list_3:
for _eet_ list_4:
var_name = str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)
var_names.append(var_name)
exec(str(_foo_)+str(_bar_)+str(_var_)+str(_eet_) + "= LpVariable("str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)+", lowBound=0, cat='Integer')")
var_value = DataBase.objects.get(column_A = str(_foo_)+str(_var_)).value
var_values.append(var_value)
obj_func = LpAffineExpression([var_names[i],var_values[i] for in in range(len(var_names))])
problem = LpProblem(name="name", sense=LpMinimise)
#Example of constraints
exec("problem += (" + str(function(a1,b1,c1)_) +str(function(a1,b1,c2)) +" >= Database_2.objects.get(column_A = z1).value")
problem += obj_func
problem.sovle()
The code works in jupyter notebook when I load the database info as a dataframe. However, I keep receiving this following error code when using in Djagno:
File "/path/to/files/prob.py", line 1610, in <module>
problem.solve()
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1913, in solve
status = solver.actualSolve(self, **kwargs)
File "/path/to/files/lib/python3.9/site-packages/pulp/apis/coin_api.py", line 137, in actualSolve
return self.solve_CBC(lp, **kwargs)
File "/path/to/files/lib/python3.9/site-packages/pulp/apis/coin_api.py", line 153, in solve_CBC
vs, variablesNames, constraintsNames, objectiveName = lp.writeMPS(
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1782, in writeMPS
return mpslp.writeMPS(self, filename, mpsSense=mpsSense, rename=rename, mip=mip)
File "/path/to/files/lib/python3.9/site-packages/pulp/mps_lp.py", line 204, in writeMPS
constrNames, varNames, cobj.name = LpProblem.normalisedNames()
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1546, in normalisedNames
_variables = self.variables()
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1624, in variables
self.addVariables(list(self.objective.keys()))
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1614, in addVariables
self.addVariable(v)
File "/path/to/files/lib/python3.9/site-packages/pulp/pulp.py", line 1603, in addVariable
if variable.hash not in self._variable_ids:
AttributeError: 'str' object has no attribute 'hash'
I have the code stored as .py file which is called views.py.
I believe it may be an issue with the namespace of the LpVariable creations. I have tried to:
define the whole problem encapsulated as a function with no entry variable and returns a dict of the solution.
Define the problem as a class with problem.create and problem.solve as methods to create the variables and solve the function.
Update the exec() code to store variables in the globals diction.
exec(str(_foo_)+str(_bar_)+str(_var_)+str(_eet_) + "= LpVariable("str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)+", lowBound=0, cat='Integer')", globals())
And alternately creating a local dict and executing the above code with locals(),local_dict .
Used LpVariable.dict
variables = LpVariable.dicts("variables", [(_foo_, _bar_, _var_, _eet_), for _foo_ in list1 for _bar_ in list2 for _var_ in list3 for _eet_ in list4], lowBound=o, cat="Integer")
This does create all the variables, however the function used in the constraints references the variables as per the name str(foo)+str(bar)+str(var)+str(eet) and not variable[i] , which then generates undefined variable errors.
As mentioned, this code does work in jupyter, I am just at a loss as to what the error may be a result of.
LpAffineExpression expects LpVariable, not var_names[i].
var_items = []
# exec(str(_foo_)+str(_bar_)+str(_var_)+str(_eet_) + "= LpVariable(" + str(_foo_)+str(_bar_)+str(_var_)+str(_eet_)+", lowBound=0, cat='Integer')")
# var_value = DataBase.objects.get(column_A = str(_foo_)+str(_var_)).value
# var_values.append(var_value)
var_value = LpVariable(var_name, lowBound=0, cat='Integer')
var_coeff = DataBase.objects.get(column_A = str(_foo_)+str(_var_)).value
var_items.append((var_value, var_coeff))
# obj_func = LpAffineExpression([var_names[i],var_values[i] for i in range(len(var_names))])
obj_func = LpAffineExpression(var_items)
Reference: https://coin-or.github.io/pulp/technical/pulp.html#pulp.LpAffineExpression
I'm trying to run a minimization problem using scipy.optimize, including a NonlinearConstraint. I really don't want to code derivatives myself, so I'm using autograd to do it. But even though I follow the exact same procedure for the arguments to minimize and to NonlinearConstraint, the first seems to work and the second doesn't.
Here's my MWE:
useconstraint = False
import autograd
import autograd.numpy as np
from scipy import optimize
def function(x): return x[0]**2 + x[1]**2
functionjacobian = autograd.jacobian(function)
functionhvp = autograd.hessian_vector_product(function)
def constraint(x): return np.array([x[0]**2 - x[1]**2])
constraintjacobian = autograd.jacobian(constraint)
constrainthvp = autograd.hessian_vector_product(constraint)
constraint = optimize.NonlinearConstraint(constraint, 1, np.inf, constraintjacobian, constrainthvp)
startpoint = [1, 2]
bounds = optimize.Bounds([-np.inf, -np.inf], [np.inf, np.inf])
print optimize.minimize(
function,
startpoint,
method='trust-constr',
jac=functionjacobian,
hessp=functionhvp,
constraints=[constraint] if useconstraint else [],
bounds=bounds,
)
When I turn useconstraint off (at the top), it works fine and minimizes at (0, 0) as expected. When I turn it on, I get the following error:
Traceback (most recent call last):
File "test.py", line 29, in <module>
bounds=bounds,
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_minimize.py", line 613, in minimize
callback=callback, **options)
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_trustregion_constr/minimize_trustregion_constr.py", line 336, in _minimize_trustregion_constr
for c in constraints]
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_constraints.py", line 213, in __init__
finite_diff_bounds, sparse_jacobian)
File "/home/heshy/.local/lib/python2.7/site-packages/scipy/optimize/_differentiable_functions.py", line 343, in __init__
self.H = hess(self.x, self.v)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/wrap_util.py", line 20, in nary_f
return unary_operator(unary_f, x, *nary_op_args, **nary_op_kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/differential_operators.py", line 24, in grad
vjp, ans = _make_vjp(fun, x)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/core.py", line 10, in make_vjp
end_value, end_node = trace(start_node, fun, x)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/tracer.py", line 10, in trace
end_box = fun(start_box)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/wrap_util.py", line 15, in unary_f
return fun(*subargs, **kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/differential_operators.py", line 88, in vector_dot_grad
return np.tensordot(fun_grad(*args, **kwargs), vector, np.ndim(vector))
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/tracer.py", line 44, in f_wrapped
ans = f_wrapped(*argvals, **kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/autograd/tracer.py", line 48, in f_wrapped
return f_raw(*args, **kwargs)
File "/home/heshy/.local/lib/python2.7/site-packages/numpy/core/numeric.py", line 1371, in tensordot
raise ValueError("shape-mismatch for sum")
ValueError: shape-mismatch for sum
What am I doing wrong? I think the issue is in the hessian_vector_product because I see hess in the error message, but I'm not sure about that.
Ok, I found the answer. This was very confusing.
The hessp argument to minimize expects a function that returns the "Hessian of objective function times an arbitrary vector p" (source). By contrast, the hess argument to NonlinearConstraint expects "A callable [that] must return the Hessian matrix of dot(fun, v)" (source).
If you interpret the first quote like I did, "Hessian of (objective function times an arbitrary vector p)", it means pretty much the same thing as "Hessian matrix of dot(fun, v)". I therefore assumed you could use the same autograd function for both.
However, the correct interpretation is "(Hessian of objective function) times an arbitrary vector p", which is completely different. The hessian_vector_product function in autograd gives the correct result for the first, but you need a different function for the second.
It's my first post, I hope it will be well done.
I'm trying to run the following ZipLine Algo with local AAPL data :
import pandas as pd
from collections import OrderedDict
import pytz
from zipline.api import order, symbol, record, order_target
from zipline.algorithm import TradingAlgorithm
data = OrderedDict()
data['AAPL'] = pd.read_csv('AAPL.csv', index_col=0, parse_dates=['Date'])
panel = pd.Panel(data)
panel.minor_axis = ['Open', 'High', 'Low', 'Close', 'Volume', 'Price']
panel.major_axis = panel.major_axis.tz_localize(pytz.utc)
print panel["AAPL"]
def initialize(context):
context.security = symbol('AAPL')
def handle_data(context, data):
MA1 = data[context.security].mavg(50)
MA2 = data[context.security].mavg(100)
date = str(data[context.security].datetime)[:10]
current_price = data[context.security].price
current_positions = context.portfolio.positions[symbol('AAPL')].amount
cash = context.portfolio.cash
value = context.portfolio.portfolio_value
current_pnl = context.portfolio.pnl
# code (this will come under handle_data function only)
if (MA1 > MA2) and current_positions == 0:
number_of_shares = int(cash / current_price)
order(context.security, number_of_shares)
record(date=date, MA1=MA1, MA2=MA2, Price=
current_price, status="buy", shares=number_of_shares, PnL=current_pnl, cash=cash, value=value)
elif (MA1 < MA2) and current_positions != 0:
order_target(context.security, 0)
record(date=date, MA1=MA1, MA2=MA2, Price=current_price, status="sell", shares="--", PnL=current_pnl, cash=cash,
value=value)
else:
record(date=date, MA1=MA1, MA2=MA2, Price=current_price, status="--", shares="--", PnL=current_pnl, cash=cash,
value=value)
#initializing trading enviroment
algo_obj = TradingAlgorithm(initialize=initialize, handle_data=handle_data)
#run algo
perf_manual = algo_obj.run(panel)
#code
#calculation
print "total pnl : " + str(float(perf_manual[["PnL"]].iloc[-1]))
buy_trade = perf_manual[["status"]].loc[perf_manual["status"] == "buy"].count()
sell_trade = perf_manual[["status"]].loc[perf_manual["status"] == "sell"].count()
total_trade = buy_trade + sell_trade
print "buy trade : " + str(int(buy_trade)) + " sell trade : " + str(int(sell_trade)) + " total trade : " + str(int(total_trade))
I was inspired by https://www.quantinsti.com/blog/introduction-zipline-python/ and https://www.quantinsti.com/blog/importing-csv-data-zipline-backtesting/.
I get this error :
Traceback (most recent call last):
File "C:/Users/main/Desktop/docs/ALGO_TRADING/_DATAS/_zipline_data_bundle /temp.py", line 51, in <module>
algo_obj = TradingAlgorithm(initialize=initialize, handle_data=handle_data)
File "C:\Python27-32\lib\site-packages\zipline\algorithm.py", line 273, in __init__
self.trading_environment = TradingEnvironment()
File "C:\Python27-32\lib\site-packages\zipline\finance\trading.py", line 99, in __init__
self.bm_symbol,
File "C:\Python27-32\lib\site-packages\zipline\data\loader.py", line 166, in load_market_data
environ,
File "C:\Python27-32\lib\site-packages\zipline\data\loader.py", line 230, in ensure_benchmark_data
last_date,
File "C:\Python27-32\lib\site-packages\zipline\data\benchmarks.py", line 50, in get_benchmark_returns
last_date
File "C:\Python27-32\lib\site-packages\pandas_datareader\data.py", line 137, in DataReader
session=session).read()
File "C:\Python27-32\lib\site-packages\pandas_datareader\base.py", line 181, in read
params=self._get_params(self.symbols))
File "C:\Python27-32\lib\site-packages\pandas_datareader\base.py", line 79, in _read_one_data
out = self._read_url_as_StringIO(url, params=params)
File "C:\Python27-32\lib\site-packages\pandas_datareader\base.py", line 90, in _read_url_as_StringIO
response = self._get_response(url, params=params)
File "C:\Python27-32\lib\site-packages\pandas_datareader\base.py", line 139, in _get_response
raise RemoteDataError('Unable to read URL: {0}'.format(url))
pandas_datareader._utils.RemoteDataError: Unable to read URL: http://www.google.com/finance/historical?q=SPY&startdate=Dec+29%2C+1989&enddate=Dec+20%2C+2017&output=csv
I don't understand : "http://www.google.com/finance/historical?q=SPY&startdate=Dec+29%2C+1989&enddate=Dec+20%2C+2017&output=csv".
I don't ask for online data request... and not 'SPY' stock but 'APPL'...
What does this error mean to you ?
Thanks a lot for your help !
C.
Only reference and workaround I found regarding this issue is here:
from pandas_datareader.google.daily import GoogleDailyReader
#property
def url(self):
return 'http://finance.google.com/finance/historical'
GoogleDailyReader.url = url
do:
pip install fix_yahoo_finance
then modify the file: zipline/lib/pythonx.x/site-packages/zipline/data/benchmarks.py
add the following two statements to the file:
import fix_yahoo_finance as yf
yf.pdr_override ()
then change following instruction:
data = pd_reader.DataReader (symbol, 'Google' first_date, last_date)
to:
data = pd_reader.get_data_yahoo(symbol,first_date, last_date)
thisfile.py
import cPickle
import gzip
import os
import numpy
import theano
import theano.tensor as T
def load_data(dataset):
f = gzip.open(dataset, 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
def shared_dataset(data_xy, borrow=True):
data_x, data_y = data_xy
shared_x = theano.shared(numpy.asarray(data_x,
dtype=theano.config.floatX),
borrow=borrow)
shared_y = theano.shared(numpy.asarray(data_y,
dtype=theano.config.floatX),
borrow=borrow)
return shared_x, T.cast(shared_y, 'int32')
test_set_x, test_set_y = shared_dataset(test_set)
valid_set_x, valid_set_y = shared_dataset(valid_set)
train_set_x, train_set_y = shared_dataset(train_set)
rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),
(test_set_x, test_set_y)]
return rval
class PCA(object):
def __init__(self):
self.param = 0
def dimemsion_transform(self, X):
m_mean = T.mean(X, axis=0)
X = X - m_mean ##################### this line makes error
return X
if __name__ == '__main__':
dataset = 'mnist.pkl.gz'
# load the MNIST data
data = load_data(dataset)
X = T.matrix('X')
m_pca = PCA()
transform = theano.function(
inputs=[],
outputs=m_pca.dimemsion_transform(X),
givens={
X: data
}
)
error showing like below
Traceback (most recent call last):
File ".../thisfile.py", line 101, in <module>
X: data
File ".../Theano/theano/compile/function.py", line 322, in function
output_keys=output_keys)
File ".../Theano/theano/compile/pfunc.py", line 443, in pfunc
no_default_updates=no_default_updates)
File ".../Theano/theano/compile/pfunc.py", line 219, in rebuild_collect_shared
cloned_v = clone_v_get_shared_updates(v, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 93, in clone_v_get_shared_updates
clone_v_get_shared_updates(i, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 93, in clone_v_get_shared_updates
clone_v_get_shared_updates(i, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 93, in clone_v_get_shared_updates
clone_v_get_shared_updates(i, copy_inputs_over)
File ".../Theano/theano/compile/pfunc.py", line 96, in clone_v_get_shared_updates
[clone_d[i] for i in owner.inputs], strict=rebuild_strict)
File ".../Theano/theano/gof/graph.py", line 242, in clone_with_new_inputs
new_inputs[i] = curr.type.filter_variable(new)
File ".../Theano/theano/tensor/type.py", line 234, in filter_variable
self=self))
TypeError: Cannot convert Type Generic (of Variable <Generic>) into Type TensorType(float64, matrix). You can try to manually convert <Generic> into a TensorType(float64, matrix).
I am making PCA function with theano but have a problem.
mean value is subtracted from MNIST data in dimension_transform in PCA class
I do not get why it gives type matching error and how do I fix it
Your problem comes from these lines:
data = load_data(dataset)
Here data is a list (as this is what load_data() returns).
transform = theano.function(
inputs=[],
outputs=m_pca.dimemsion_transform(X),
givens={
X: data
}
)
And here you pass it as a value. You have to extract the item you want from the return value of load_data() like so:
[(train_set_x, train_set_y), (valid_set_x, valid_set_y),
(test_set_x, test_set_y)] = load_data(dataset)
and then use
givens={
X: train_set_x
}
or one of the other values.
Hi guys I want ask about json.dump
I use scikit to tune some method with parameters and I want dump it to json, but I got some error here :
I have parameter with method KNN:
KNeighborsClassifier(algorithm=u'auto', leaf_size=30, metric=u'manhattan',
metric_params=None, n_jobs=-1, n_neighbors=300, p=2,
weights=u'distance')
But got error like this:
Traceback (most recent call last):
File "jamu.py", line 1018, in <module>
main(argv)
File "jamu.py", line 863, in main
json.dumps(meta_clf, f)
File "C:\Python27\lib\json\__init__.py", line 250, in d
sort_keys=sort_keys, **kw).encode(obj)
File "C:\Python27\lib\json\encoder.py", line 207, in en
chunks = self.iterencode(o, _one_shot=True)
File "C:\Python27\lib\json\encoder.py", line 270, in it
return _iterencode(o, 0)
File "C:\Python27\lib\json\encoder.py", line 184, in de
raise TypeError(repr(o) + " is not JSON serializable"
TypeError: KNeighborsClassifier(algorithm=u'auto', leaf_s
tan',
metric_params=None, n_jobs=-1, n_neighbors=300
weights=u'distance') is not JSON serializable
Is anything wrong with my code?
It is what it says on the tin, KNeighborsClassifier cannot be serialised with json.
You'll have to use a different way to serialise a model. For example you can use joblib:
from sklearn.external import joblib
# Suppose your KNeighborsClassifier model is called knn
joblib.dump(knn, 'some/kind/of/path/knn.joblib')
Loading a model is equally simple:
knn = joblib.load('some/kind/of/path/knn.joblib')
Check the joblib docs for what else it is good for.
finally i use this way:
meta_clf = KNeighborsClassifier(algorithm=u'auto', leaf_size=30, metric=u'manhattan',
metric_params=None, n_jobs=-1, n_neighbors=300, p=2,
weights=u'distance')
def print_to_json(meta_clf):
meta_clf_str = str(meta_clf)
meta_clf_str = meta_clf_str[meta_clf_str.index("(") + 1:meta_clf_str.rindex(")")]
meta_clf_str = meta_clf_str.replace('\n ', '')
meta_clf_str = meta_clf_str.replace(' ', '')
meta_clf_str = meta_clf_str.replace('=u\'', '=\'')
meta_clf_str = meta_clf_str.replace('\'', '')
meta_clf_str_list = meta_clf_str.split(',')
meta_clf_str_list_len = len(meta_clf_str_list)
meta_clf_str_lists = []
params = {}
for x in meta_clf_str_list:
meta_clf_str_list = x.split('=')
if meta_clf_str_list[1].isdigit() == True:
meta_clf_str_list[1] = int(meta_clf_str_list[1])
meta_clf_str_lists.append(meta_clf_str_list)
params[meta_clf_str_list[0]] = meta_clf_str_list[1]
return params
it's enough for me.
thanks for the asnwer thomas, i appreciate it.