Related
I have the code using keras 1.2 and tensorflow 1.1. I have run it but with error
import numpy as np
import keras
from keras import backend as K
from keras import initializers
from keras.models import Sequential, Model, load_model, save_model
from keras.layers.core import Dense, Lambda, Activation
from keras.layers import Embedding, Input, Dense, Multiply, Reshape, Flatten
from keras.optimizers import Adagrad, Adam, SGD, RMSprop
from keras.regularizers import l2
from sklearn.metrics import average_precision_score
from sklearn.metrics import auc
def init_normal(shape, name=None):
return initializers.lecun_uniform(seed=None)
def get_model(num_a, num_b, num_c, dim, regs=[0,0,0]):
a = Input(shape=(1,), dtype='int32', name = 'a')
b = Input(shape=(1,), dtype='int32', name = 'b')
c = Input(shape=(1,), dtype='int32', name = 'c')
Embedding_a = Embedding(input_dim = num_a, output_dim = dim,
embeddings_initializer='uniform', W_regularizer = l2(regs[0]), input_length=1)
Embedding_b = Embedding(input_dim = num_b, output_dim = dim,
embeddings_initializer='uniform', W_regularizer = l2(regs[1]), input_length=1)
Embedding_c = Embedding(input_dim = num_c, output_dim = dim,
embeddings_initializer='uniform', W_regularizer = l2(regs[2]), input_length=1)
a_latent = Flatten()(Embedding_a(a))
b_latent = Flatten()(Embedding_b(b))
c_latent = Flatten()(Embedding_c(c))
predict_vector = Multiply()([a_latent, b_latent, b_latent])
prediction = Dense(1, activation='sigmoid', init='lecun_uniform', name = 'prediction')(predict_vector)
model = Model(input=[a, b, c], output=prediction)
return model
def evaluate_model(model, test_pos, test_neg):
global _model
global _test_pos
global _test_neg
_model = model
_test_pos = test_pos
_test_neg = test_neg
print(_test_neg)
a, b, c, labels = [],[],[],[]
for item in _test_pos:
a.append(item[0])
b.append(item[1])
c.append(item[2])
labels.append(1)
for item in _test_neg:
a.append(item[0])
b.append(item[1])
c.append(item[2])
labels.append(0)
a = np.array(a)
b = np.array(b)
c = np.array(c)
predictions = _model.predict([a, b, c],
batch_size=100, verbose=0)
return average_precision_score(labels, predictions), auc(labels, predictions)
model = get_model(4, 8, 12, 2, [0,0,0])
model.compile(optimizer=Adam(lr=0.001), loss='binary_crossentropy')
pos_test = [[0, 0, 2], [4, 8, 8], [2, 5, 4], [0, 0, 0]]
neg_test = [[3, 3, 2], [2, 1, 8], [1, 4, 1], [3, 3, 12]]
aupr, auc = evaluate_model(model, pos_test, neg_test)
print(aupr, auc)
However, It give me error:any way to fix it?
InvalidArgumentError (see above for traceback): indices[1,0] = 4 is not in [0, 4)
[[Node: embedding_4/embedding_lookup = Gather[Tindices=DT_INT32, Tparams=DT_FLOAT, _class=["loc:#embedding_4/embeddings"], validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](embedding_4/embeddings/read, _recv_a_1_0)]]
The problem is, you defined embedding input_dim as 4, 8 and 12 while it should be is 5, 9, 13. Because input_dim in embedding should be max_index + 1. It is also clearly mentioned in Keras docs:
Size of the vocabulary, i.e. maximum integer index + 1.
How to fix the issue?
Change get_model method to:
model = get_model(5, 9, 13, 2, [0, 0, 0])
Or alternatively change index of data to:
pos_test = [[0, 0, 2], [3, 7, 7], [2, 5, 4], [0, 0, 0]]
neg_test = [[3, 3, 2], [2, 1, 7], [1, 4, 1], [3, 3, 11]]
I have a simple Tensorflow model which is written in python and could be trained smoothly. Here is the model I'm using:
import tensorflow as tf from tensorflow.python.framework
import ops from tensorflow.python.framework
import dtypes
import random import numpy as np
NUM_CLASSES = 102 IMAGE_HEIGHT = 224 IMAGE_WIDTH = 224 BATCH_SIZE = 25 NUM_CHANNELS = 3 LEARNING_RATE = 0.0001
with tf.Session() as sess:
images_placeholder = tf.placeholder (tf.float32,
shape=(BATCH_SIZE, IMAGE_HEIGHT,
IMAGE_WIDTH, NUM_CHANNELS), name="input") labels_placeholder = tf.placeholder (tf.float32,
shape=(BATCH_SIZE), name="label")
with tf.name_scope("conv1_1") as scope: kernel = tf.Variable (tf.truncated_normal([3, 3, 3, 64], dtype=tf.float32, stddev=1e-2),
name="weights") conv = tf.nn.conv2d (images_placeholder, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable (tf.constant(0.0, shape=[64], dtype=tf.float32),
trainable=True, name='biases') out = tf.nn.bias_add (conv, biases) conv1_1 = tf.nn.relu (out, name=scope)
pool1 = tf.nn.max_pool (conv1_1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
with tf.name_scope('conv2_1') as scope: kernel = tf.Variable (tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32,
stddev=1e-2), name='weights') conv = tf.nn.conv2d (pool1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable (tf.constant(0.0, shape=[128], dtype=tf.float32),
trainable=True, name='biases') out = tf.nn.bias_add (conv, biases) conv2_1 = tf.nn.relu (out, name=scope)
pool2 = tf.nn.max_pool (conv2_1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
with tf.name_scope('conv3_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32,
stddev=1e-2), name='weights') conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) conv3_1 = tf.nn.relu(out, name=scope)
pool3 = tf.nn.max_pool (conv3_1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool3')
with tf.name_scope('conv4_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512], dtype=tf.float32,
stddev=1e-2), name='weights') conv = tf.nn.conv2d(pool3, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) conv4_1 = tf.nn.relu(out, name=scope)
pool4 = tf.nn.max_pool (conv4_1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
with tf.name_scope('mentee_conv5_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
stddev=1e-2), name='weights') conv = tf.nn.conv2d(pool4, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) conv5_1 = tf.nn.relu(out, name=scope)
pool5 = tf.nn.max_pool (conv5_1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool5')
with tf.name_scope('fc1') as scope: shape = int(np.prod(pool5.get_shape()[1:])) fc1w = tf.Variable(tf.truncated_normal([shape, 4096], dtype=tf.float32,
stddev=1e-2), name='weights') fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases') pool5_flat = tf.reshape(pool5, [-1, shape]) fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b) fc1 = tf.nn.relu(fc1l)
fc1 = tf.nn.dropout(fc1, 0.5)
labels = tf.to_int64(labels_placeholder) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits (labels=labels,
logits=fc1, name="xentropy") loss = tf.reduce_mean (cross_entropy, name='loss')
optimizer = tf.train.AdamOptimizer (LEARNING_RATE) global_step = tf.Variable (0, name='global_step', trainable=False) train_op = optimizer.minimize (loss, global_step=global_step, name="train")
init = tf.initialize_variables (tf.all_variables(), name='init_all_vars_op') tf.train.write_graph (sess.graph_def, "models/", "graph.pb", as_text=False)
I'm exporting the model into the protobuf version and then load it into my C++ code, in order to train the model using C++ API instead of Python. Unfortunately I'll get an error regarding the SparseSoftmaxCrossEntropy operation as:
E tensorflow/core/common_runtime/executor.cc:594] Executor failed to create kernel. Invalid argument: NodeDef mentions attr 'message' not in Op<name=PreventGradient; signature=input:T -> output:T; attr=T:type>; NodeDef: gradients/xentropy/xentropy_grad/PreventGradient = PreventGradient[T=DT_FLOAT, message="Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\'s interaction with tf.gradients()", _device="/job:localhost/replica:0/task:0/cpu:0"](xentropy/xentropy:1)
I have checked tensorflow source code and realized the gradient caclulation for this operation has not been implemented. But I'm wondering how python version succeeds with no issue, but in C++ it tries to calculate the gradient?
I'm doing text matching using tensorflow, before i call tf.nn.embedding_lookup(word_embedding_matrix, combine_result), I have to combine some words from 2 sentence(get m words from sentence S1 and also get m words from sentence S2, then combine them together as "combine_result"), but when the code gose to tf.nn.embedding_lookup(word_embedding_matrix, combine_result) it gives me the error:
ValueError: Tensor("Reshape_7:0", shape=(1, 6), dtype=int32) must be
from the same graph as Tensor("word_embedding_matrix:0", shape=(26320,
50), dtype=float32_ref).
the code is as bellow:
import tensorflow as tf
import numpy as np
import os
import time
import datetime
import data_helpers
NUM_CLASS = 2
SEQUENCE_LENGTH = 47
# Placeholders for input, output and dropout
input_x = tf.placeholder(tf.int32, [None, 2, SEQUENCE_LENGTH], name="input_x")
input_y = tf.placeholder(tf.float32, [None, NUM_CLASS], name="input_y")
dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")
def n_grams(text, window_size):
text_left_window = []
# text_left_window = tf.convert_to_tensor(text_left_window, dtype=tf.int32)
for z in range(SEQUENCE_LENGTH-2):
text_left = tf.slice(text, [z], [window_size])
text_left_window = tf.concat(0, [text_left_window, text_left])
text_left_window = tf.reshape(text_left_window, [-1, window_size])
return text_left_window
def inference(vocab_size, embedding_size, batch_size, slide_window_size, conv_window_size):
# # Embedding layer
word_embedding_matrix = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0),
name="word_embedding_matrix")
# convo_unit = tf.Variable(tf.random_uniform([slide_window_size*2, ], -1.0, 1.0), name="convo_unit")
text_comp_result = []
for x in range(batch_size):
# input_x_slice_reshape = [[1 1 1...]
# [2 2 2...]]
input_x_slice = tf.slice(input_x, [x, 0, 0], [1, 2, SEQUENCE_LENGTH])
input_x_slice_reshape = tf.reshape(input_x_slice, [2, SEQUENCE_LENGTH])
# text_left_flat: [294, 6, 2, 6, 2, 57, 2, 57, 147, 57, 147, 5, 147, 5, 2,...], length = SEQUENCE_LENGTH
# text_right_flat: [17, 2, 2325, 2, 2325, 5366, 2325, 5366, 81, 5366, 81, 1238,...]
text_left = tf.slice(input_x_slice_reshape, [0, 0], [1, SEQUENCE_LENGTH])
text_left_flat = tf.reshape(text_left, [-1])
text_right = tf.slice(input_x_slice_reshape, [1, 0], [1, SEQUENCE_LENGTH])
text_right_flat = tf.reshape(text_right, [-1])
# extract both text.
# text_left_window: [[294, 6, 2], [6, 2, 57], [2, 57, 147], [57, 147, 5], [147, 5, 2],...]
# text_right_window: [[17, 2, 2325], [2, 2325, 5366], [2325, 5366, 81], [5366, 81, 1238],...]
text_left_window = n_grams(text_left_flat, slide_window_size)
text_right_window = n_grams(text_right_flat, slide_window_size)
text_left_window_sha = text_left_window.get_shape()
print 'text_left_window_sha:', text_left_window_sha
# composite the slice
text_comp_list = []
# text_comp_list = tf.convert_to_tensor(text_comp_list, dtype=tf.float32)
for l in range(SEQUENCE_LENGTH-slide_window_size+1):
text_left_slice = tf.slice(text_left_window, [l, 0], [1, slide_window_size])
text_left_slice_flat = tf.reshape(text_left_slice, [-1])
for r in range(SEQUENCE_LENGTH-slide_window_size+1):
text_right_slice = tf.slice(text_right_window, [r, 0], [1, slide_window_size])
text_right_slice_flat = tf.reshape(text_right_slice, [-1])
# convo_unit = [294, 6, 2, 17, 2, 2325]
convo_unit = tf.concat(0, [text_left_slice_flat, text_right_slice_flat])
convo_unit_reshape = tf.reshape(convo_unit, [-1, slide_window_size*2])
# convo_unit_shape_val = convo_unit_reshape.get_shape()
# print 'convo_unit_shape_val:', convo_unit_shape_val
embedded_chars = tf.nn.embedding_lookup(word_embedding_matrix, convo_unit_reshape)
embedded_chars_expanded = tf.expand_dims(embedded_chars, -1)
...
could please someone help me? Thank you very much!
Yaroslav answered in a comment above - moving to an answer:
This error happens when you create new default graph. Try to do tf.reset_default_graph() before the computation and not create any more graphs (i.e., calls to tf.Graph)
import csv
from itertools import izip
if l > 0:
for i in range(0,l):
combined.append(str(questionList[i]).encode('utf-8') + str(viewList[i]).encode('utf-8'))
# viewcsv.append(str(viewList[i]).encode('utf-8'))
# quescsv.append(str(questionList[i]).encode('utf-8'))
with open('collect.csv', 'a') as csvfile:
spamwriter = csv.writer(csvfile, delimiter='\n')
spamwriter.writerow(combined)
# spamwriter.writerows(izip(quescsv, viewcsv))
return 1
else:
return 0
I need to generate a csv file and flood it with data from 2 or more lists into separate columns and not a single column. Currently I'm trying to combine two lists in one list(combined) and use this as input for writing, but I haven't got desired o/p.
I have tried many things including the fieldnames way,izip way, but in vain.
Eg:
questionList viewList
4 3 views
5 0 views
The numbers used are just for example.
Probably, you need something like this:
import csv
X = [1, 2, 3, 4, 5]
Y = [2, 3, 5, 7, 11]
Z = ['two', 'three', 'five', 'seven', 'eleven']
with open('collect.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
for row in zip(X, Y, Z):
writer.writerow(row)
import csv
X = [1, 2, 3, 4, 5]
Y = [2, 3, 5, 7, 11]
Z = ['two', 'three', 'five', 'seven', 'eleven']
with open('collect.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(X)
writer.writerow(Y)
writer.writerow(Z)
My code is currently written as:
convert = {0:0,1:1,2:2,3:3,4:0,5:1,6:2,7:1}
rows = [[convert[random.randint(0,7)] for _ in range(5)] for _ in range(5)]
numgood = 25 - rows.count(0)
print numgood
>> 25
It always comes out as 25, so it's not just that rows contains no 0's.
Have you printed rows?
It's [[0, 1, 0, 0, 2], [1, 2, 0, 1, 2], [3, 1, 1, 1, 1], [1, 0, 0, 1, 0], [0, 3, 2, 0, 1]], so you have a nested list there.
If you want to count the number of 0's in those nested lists, you could try:
import random
convert = {0:0, 1:1, 2:2, 3:3, 4:0, 5:1, 6:2, 7:1}
rows = [[convert[random.randint(0, 7)] for _ in range(5)] for _ in range(5)]
numgood = 25 - sum(e.count(0) for e in rows)
print numgood
Output:
18
rows doesn't contain any zeroes; it contains lists, not integers.
>>> row = [1,2,3]
>>> type(row)
<type 'list'>
>>> row.count(2)
1
>>> rows = [[1,2,3],[4,5,6]]
>>> rows.count(2)
0
>>> rows.count([1,2,3])
1
To count the number of zeroes in any of the lists in rows, you could use a generator expression:
>>> rows = [[1,2,3],[4,5,6], [0,0,8]]
>>> sum(x == 0 for row in rows for x in row)
2
You could also use numpy:
import numpy as np
import random
convert = {0:0,1:1,2:2,3:3,4:0,5:1,6:2,7:1}
rows = [[convert[random.randint(0,7)] for _ in range(5)] for _ in range(5)]
numgood = 25 - np.count_nonzero(rows)
print numgood
Output:
9