I followed the official tutorial of the tensorflow website: https://www.tensorflow.org/extend/adding_an_op
There is also described how to call the gradient of the example ZeroOut in the tutorial that I want to try in this short code snippet underneath.
I have found the code here: https://github.com/MatteoRagni/tf.ZeroOut.gpu
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import sparse_ops
zero_out_module = tf.load_op_library('./libzeroout.so')
#ops.RegisterGradient("ZeroOut")
def _zero_out_grad(op, grad):
to_zero = op.inputs[0]
shape = array_ops.shape(to_zero)
index = array_ops.zeros_like(shape)
first_grad = array_ops.reshape(grad, [-1])[0]
to_zero_grad = sparse_ops.sparse_to_dense([index], shape, first_grad, 0)
return [to_zero_grad] # List of one Tensor, since we have one input
t_in = tf.placeholder(tf.int32, [None,None])
ret = zero_out_module.zero_out(t_in)
grad = tf.gradients(ys=tf.reduce_sum(ret), xs=t_in)
with tf.Session(''):
feed_dict = {t_in: [[1, 2], [3, 4]]}
print "ret val: ", ret.eval(feed_dict=feed_dict)
print "grad: ", grad
print "grad: ", grad.eval(feed_dict=feed_dict)
I got this error ...
AttributeError: 'list' object has no attribute 'eval'
... but I can do ret.eval().
Why I cant call grad.eval()? I want to see these values inside the grad tensor. How to debug gradient?
Answer to old question
The implementation
def _zero_out_grad(op, *grads):
topdiff = grads[0]
bottom = op.inputs[0]
shape = array_ops.shape(bottom)
index = array_ops.zeros_like(shape)
first_grad = array_ops.reshape(topdiff, [-1])[0]
to_zero_grad = sparse_ops.sparse_to_dense([index], shape, first_grad, 0)
return to_zero_grad
works quite nicely here. Are you sure "#ops.RegisterGradient("ZeroOut")" is executed before the tf.Session()?
Usually the
zero_out_module = tf.load_op_library('./libzeroout.so')
#ops.RegisterGradient("ZeroOut")
def _zero_out_grad(op, grad):
# ...
is placed in a different file and just imported. A full working example even with the recent TensorFlow version is here.
Answer to completely changed question
Your gradient function returns a list and a Python list has no 'eval()'. Try either:
grad = tf.gradients(ys=tf.reduce_sum(ret), xs=t_in)[0]
Or follow best practice and use
grad = tf.gradients(ys=tf.reduce_sum(ret), xs=t_in)
with tf.Session() as sess:
sess.run(grad, feed_dict=feed_dict)
Please do not change your entire question
Related
I am trying to parallelize a penalized linear model using the multiprocessing library in python.
I created a function that solves my model:
from __future__ import division
import numpy as np
from cvxpy import *
def lm_lasso_solver(x, y, lambda1):
n = x.shape[0]
m = x.shape[1]
lambda1_param = Parameter(sign="positive")
betas_var = Variable(m)
response = dict(model='lm', penalization='l')
response["parameters"] = {"lambda_vector": lambda1}
lasso_penalization = lambda1_param * norm(betas_var, 1)
lm_penalization = 0.5 * sum_squares(y - x * betas_var)
objective = Minimize(lm_penalization + lasso_penalization)
problem = Problem(objective)
lambda1_param.value = lambda1
try:
problem.solve(solver=ECOS)
except:
try:
problem.solve(solver=CVXOPT)
except:
problem.solve(solver=SCS)
beta_sol = np.asarray(betas_var.value).flatten()
response["solution"] = beta_sol
return response
In this function x is a matrix of predictors and y is the response variable. lambda1 is the parameter that must be optimized, and so, is the parameter that I want to parallelize. I saved this script in a python file called "ms.py"
Then I created another python file called "parallelization.py" and in that file I defined the following:
import multiprocessing as mp
import ms
import functools
def myFunction(x, y, lambda1):
pool = mp.Pool(processes=mp.cpu_count())
results = pool.map(functools.partial(ms.lm_lasso_solver, x=x, y=y), lambda1)
return results
So the idea was now, on the python interpreter, execute:
from sklearn.datasets import load_boston
boston = load_boston()
x = boston.data
y = boston.target
runfile('parallelization.py')
lambda_vector = np.array([1,2,3])
myFunction(x, y, lambda_vector)
But when I do this, I get the following error message:
The problem is on the line:
results = pool.map(functools.partial(ms.lm_lasso_solver, x=x, y=y), lambda1)
You are calling the functools.partial() method with keyworded arguments whereas in your lm_lasso_solver method, you don't define them as keyworded arguments. You should call it with x and y as positional arguments as follows:
results = pool.map(functools.partial(ms.lm_lasso_solver, x, y), lambda1)
or simply use the apply_async() method the pool object :
results = pool.apply_async(ms.lm_lasso_solver, args=[x, y, lambda1])
I am new to tensorflow and was playing around with a deep learning network. I wanted to do a custom rounding off on all the weights after each iteration. As the round function in tensorflow library doesn't give you the option to round the values down to a certain number of decimal points.
So I wrote this
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
np_prec = lambda x: np.round(x,3).astype(np.float32)
def tf_prec(x,name=None):
with ops.name_scope( "d_spiky", name,[x]) as name:
y = tf.py_func(np_prec,
[x],
[tf.float32],
name=name,
stateful=False)
return y[0]
with tf.Session() as sess:
x = tf.constant([0.234567,0.712,1.2,1.7])
y = tf_prec(x)
y = tf_prec(x)
tf.global_variables_initializer
print(x.eval(), y.eval())
The output I got was this
[ 0.234567 0.71200001 1.20000005 1.70000005] [ 0.235 0.71200001 1.20000005 1.70000005]
So the custom rounding off worked only on the first item in the tensor and I am not sure about what I am doing wrong. Thanks in advance.
The error here because of the following line,
np_prec = lambda x: np.round(x,3).astype(np.float32)
you are casting the output to np.float32. You can verify the error by the following code,
print(np.round([0.234567,0.712,1.2,1.7], 3).astype(np.float32)) #prints [ 0.235 0.71200001 1.20000005 1.70000005]
The default output of np.round is float64. Moreover, you also have to change the Tout argument in tf.py_func to float64.
I have given the following code with the above fix and commented where necessary.
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
np_prec = lambda x: np.round(x,3)
def tf_prec(x,name=None):
with ops.name_scope( "d_spiky", name,[x]) as name:
y = tf.py_func(np_prec,
[x],
[tf.float64], #changed this line to tf.float64
name=name,
stateful=False)
return y[0]
with tf.Session() as sess:
x = tf.constant([0.234567,0.712,1.2,1.7],dtype=np.float64) #specify the input data type np.float64
y = tf_prec(x)
y = tf_prec(x)
tf.global_variables_initializer
print(x.eval(), y.eval())
Hope this helps.
I need to test a call to gzip.open, but I need it to supply it with an actual test file with test data in it to read. I've seen several very similar questions, but none of them work as expected.
This is the code I'm testing:
with gzip.open(local_file_path,'r') as content:
for line in content:
try:
if line.startswith('#'):
continue
line_data = line.split('\t')
request_key = line_data[LINE_FORMAT['date']]
request_key += '-' + line_data[LINE_FORMAT['time']][:-3]
request_key += '-' + line_data[LINE_FORMAT['source_ip']]
if request_key in result.keys():
result[request_key] += 1
else:
result[request_key] = 1
num_requests += 1
except Exception, e:
print ("[get_outstanding_requesters] \t\tError to process line: %s"%line)
I think the problem is related to the issues discussed here because the code treats the file as an iterator, but none of the workarounds discussed have worked for me.
I've tried variations on this:
test_data = open('test_access_log').read()
m = mock.mock_open(read_data=test_data)
m.return_value.__iter__ = lambda self:self
m.return_value.__next__ = lambda self: self.readline()
with mock.patch('gzip.open', m):
with gzip.open('asdf') as f:
for i in f:
print i
Which results in:
TypeError: iter() returned non-iterator of type 'MagicMock'
I'm using Python 2.7. I'm tearing my hair out over this one. Is my only solution to forget trying to use an iterator (actual files can be very large, which is why I'm trying to avoid doing so?)
This is working:
import unittest
import mock
test_data = open('test_access_log').read()
m = mock.mock_open(read_data=test_data)
m.return_value.__iter__.return_value = test_data.splitlines()
with mock.patch('gzip.open', m):
with gzip.open('test_access_log') as f:
for i in f:
print i
Thanks to bash-shell.net
import tensorflow as tf
array = tf.Variable(tf.random_normal([10]))
i = tf.constant(0)
l = []
def cond(i,l):
return i < 10
def body(i,l):
temp = tf.gather(array,i)
l.append(temp)
return i+1,l
index,list_vals = tf.while_loop(cond, body, [i,l])
I want to process a tensor array in the similar way as described in the above code. In the body of the while loop I want to process the array by element by element basis to apply some function. For demonstration, I have given a small code snippet. However, it is giving an error message as follows.
ValueError: Number of inputs and outputs of body must match loop_vars: 1, 2
Any help in resolving this is appreciated.
Thanks
Citing the documentation:
loop_vars is a (possibly nested) tuple, namedtuple or list
of tensors that is passed to both cond and body
You cannot pass regular python array as a tensor. What you can do, is:
i = tf.constant(0)
l = tf.Variable([])
def body(i, l):
temp = tf.gather(array,i)
l = tf.concat([l, [temp]], 0)
return i+1, l
index, list_vals = tf.while_loop(cond, body, [i, l],
shape_invariants=[i.get_shape(),
tf.TensorShape([None])])
The shape invariants are there, because normally tf.while_loop expects the shapes of tensors inside while loop won't change.
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(list_vals)
Out: array([-0.38367489, -1.76104736, 0.26266089, -2.74720812, 1.48196387,
-0.23357525, -1.07429159, -1.79547787, -0.74316853, 0.15982138],
dtype=float32)
TF offers a TensorArray to deal with such cases. From the doc,
Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays.
This class is meant to be used with dynamic iteration primitives such as while_loop and map_fn. It supports gradient back-propagation via special "flow" control flow dependencies.
Here is an example,
import tensorflow as tf
array = tf.Variable(tf.random_normal([10]))
step = tf.constant(0)
output = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)
def cond(step, output):
return step < 10
def body(step, output):
output = output.write(step, tf.gather(array, step))
return step + 1, output
_, final_output = tf.while_loop(cond, body, loop_vars=[step, output])
final_output = final_output.stack()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(sess.run(final_output))
I am currently coding an Multiple Gradient Descent algorithm, where I use kriged functions.
My problem is that I can't find how to obtain the gradient of the kriged function (I tried to use linearize but I don't know how to make it work).
from __future__ import print_function
from six import moves
from random import shuffle
import sys
import numpy as np
from numpy import linalg as LA
import math
from openmdao.braninkm import F, G, DF, DG
from openmdao.api import Group, Component,IndepVarComp
from openmdao.api import MetaModel
from openmdao.api import KrigingSurrogate, FloatKrigingSurrogate
def rand_lhc(b, k):
# Calculates a random Latin hypercube set of n points in k dimensions within [0,n-1]^k hypercube.
arr = np.zeros((2*b, k))
row = list(moves.xrange(-b, b))
for i in moves.xrange(k):
shuffle(row)
arr[:, i] = row
return arr/b*1.2
class TrigMM(Group):
''' FloatKriging gives responses as floats '''
def __init__(self):
super(TrigMM, self).__init__()
# Create meta_model for f_x as the response
F_mm = self.add("F_mm", MetaModel())
F_mm.add_param('X', val=np.array([0., 0.]))
F_mm.add_output('f_x:float', val=0., surrogate=FloatKrigingSurrogate())
# F_mm.add_output('df_x:float', val=0., surrogate=KrigingSurrogate().linearize)
#F_mm.linearize('X', 'f_x:float')
#F_mm.add_output('g_x:float', val=0., surrogate=FloatKrigingSurrogate())
print('init ok')
self.add('p1', IndepVarComp('X', val=np.array([0., 0.])))
self.connect('p1.X','F_mm.X')
# Create meta_model for f_x as the response
G_mm = self.add("G_mm", MetaModel())
G_mm.add_param('X', val=np.array([0., 0.]))
G_mm.add_output('g_x:float', val=0., surrogate=FloatKrigingSurrogate())
#G_mm.add_output('df_x:float', val=0., surrogate=KrigingSurrogate().linearize)
#G_mm.linearize('X', 'g_x:float')
self.add('p2', IndepVarComp('X', val=np.array([0., 0.])))
self.connect('p2.X','G_mm.X')
from openmdao.api import Problem
prob = Problem()
prob.root = TrigMM()
prob.setup()
u=4
v=3
#training avec latin hypercube
prob['F_mm.train:X'] = rand_lhc(20,2)
prob['G_mm.train:X'] = rand_lhc(20,2)
#prob['F_mm.train:X'] = rand_lhc(10,2)
#prob['G_mm.train:X'] = rand_lhc(10,2)
#prob['F_mm.linearize:X'] = rand_lhc(10,2)
#prob['G_mm.linearize:X'] = rand_lhc(10,2)
datF=[]
datG=[]
datDF=[]
datDG=[]
for i in range(len(prob['F_mm.train:X'])):
datF.append(F(np.array([prob['F_mm.train:X'][i]]),u))
#datG.append(G(np.array([prob['F_mm.train:X'][i]]),v))
data_trainF=np.fromiter(datF,np.float)
for i in range(len(prob['G_mm.train:X'])):
datG.append(G(np.array([prob['G_mm.train:X'][i]]),v))
data_trainG=np.fromiter(datG,np.float)
prob['F_mm.train:f_x:float'] = data_trainF
#prob['F_mm.train:g_x:float'] = data_trainG
prob['G_mm.train:g_x:float'] = data_trainG
Are you going to be writing a Multiple Gradient Descent driver? If so, then OpenMDAO calculates the gradient from a param to an output at the Problem level using the calc_gradient method.
If you take a look at the source code for the pyoptsparse driver:
https://github.com/OpenMDAO/OpenMDAO/blob/master/openmdao/drivers/pyoptsparse_driver.py
The _gradfunc method is a callback function that returns the gradient of the constraints and objectives with respect to the design variables. The Metamodel component has built-in analytic gradients for all (I think) of our surrogates, so you don't even have to declare any there.
If this isn't what you are trying to do, then I may need a little more information about your application.