I am currently working on a Neural network that maps stft audio files to MFCC features..
But keep for some reason keep getting a validation_loss around 200.
Which compared desired range is (-100,100).. so the output the NN outputs is not desirable..
I tried different implementation..
Simple NN 1-layer, 2-layer, 3-layer => wiht no difference.
CNN , 1-layer, 2-layer => same result
tried with random forrest => never finished, memory allocation error.
So...
What am I doing wrong?
I tried normalizing the input and output, but the same error came back when i unormalized the dataset.. so...
the shape of the input is (x,2050) and the shape of the output (x,13)...
Any idea on why I am getting so poor results?..
Normalization:
def numpy_minmax(X):
print X.min()
print X.max()
xmin = X.min()
return (2*(X - xmin) / (X.max() - xmin)-1)*0.9
def numpy_unorm(x):
xmax = 109.2991
xmin = -97.23664
return x*(xmax-xmin)+xmin
files_train_path = [dnn_train+f for f in listdir(dnn_train) if isfile(join(dnn_train, f))]
files_test_path = [dnn_test+f for f in listdir(dnn_test) if isfile(join(dnn_test, f))]
files_train_name = [f for f in listdir(dnn_train) if isfile(join(dnn_train, f))]
files_test_name = [f for f in listdir(dnn_test) if isfile(join(dnn_test, f))]
os.chdir(dnn_train)
#train_name = generate_list_of_names_data(files_train_path)
#print train_name
train_data, train_output_data, max = load_sound_files(files_train_path)
#sys.exit()
#train_data_real, train_data_img = split_real_img(train_data)
#print train_data
train_set_data_vstacked = np.vstack(train_data)
train_set_output_vstacked = np.vstack(train_output_data)
print train_set_data_vstacked.shape
train_set_data_vstacked_normalized = numpy_minmax(train_set_data_vstacked)
train_set_output_vstacked_normalized = numpy_minmax(train_set_output_vstacked)
Currently network structure options tried:
############################### Training setup ##################################
#Define 10 folds:
seed = 7
np.random.seed(seed)
kfold = KFold(n_splits=10, shuffle=False, random_state=None)
print "Splits"
cvscores_acc = []
cvscores_loss = []
hist = []
i = 0
#train_set_data_vstacked_normalized_reshaped = np.reshape(train_set_data_vstacked_normalized,train_set_data_vstacked_normalized.shape+(1,))
#train_set_output_vstacked_normalized_reshaped = np.reshape(train_set_output_vstacked_normalized,train_set_output_vstacked_normalized.shape+(1,))
for train, test in kfold.split(train_set_data_vstacked_normalized):
print "Model definition!"
model = Sequential()
#act = PReLU(init='normal', weights=None)
model.add(Dense(output_dim=2050,input_dim=2050, init="normal", activation='relu'))
#act1 = PReLU(init='normal', weights=None)
#act2 = PReLU(init='normal', weights=None)
model.add(Dense(output_dim=2050, input_dim=2050, init="he_normal",activation='tanh'))
model.add(Dense(output_dim=13, input_dim=2050, init="he_normal",activation='tanh'))
model.add(Lambda(lambda x: numpy_unorm(x)))
#model.add(ELU(100))
#model.add(Convolution1D(13, 3, border_mode='same', input_shape=(2050,1)))
print "Compiling"
#rms_opt = keras.optimizers.RMSprop(lr=0.01, rho=0.9, epsilon=1e-08, decay=0.0)
model.compile(loss='mean_squared_error', optimizer="RMSprop")
print "Compile done! "
print '\n'
print "Train start"
reduce_lr=ReduceLROnPlateau(monitor='val_loss', factor=0.01, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0.000000000000000001)
stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
log=csv_logger = CSVLogger('training_'+str(i)+'.csv')
hist_current = model.fit(train_set_data_vstacked_normalized[train],
train_set_output_vstacked[train],
shuffle=False,
validation_data=(train_set_data_vstacked_normalized[test],train_set_output_vstacked[test]),
validation_split=0.1,
nb_epoch=150,
verbose=1,
callbacks=[reduce_lr,log,stop])
hist.append(hist_current)
Convolutional:
#Each frame length = 118
#Each feature length = 13
############################### Training setup ##################################
#Define 10 folds:
seed = 7
np.random.seed(seed)
kfold = KFold(n_splits=10, shuffle=False, random_state=None)
print "Splits"
cvscores_acc = []
cvscores_loss = []
hist = []
i = 0
#train_set_data_vstacked_normalized_reshaped = np.reshape(train_set_data_vstacked_normalized,train_set_data_vstacked_normalized.shape+(1,))
#train_set_output_vstacked_normalized_reshaped = np.reshape(train_set_output_vstacked_normalized,train_set_output_vstacked_normalized.shape+(1,))
train_set_data_vstacked_reshaped = train_set_data_vstacked[:,newaxis,:]
train_set_output_vstacked_reshaped = train_set_output_vstacked[:,newaxis,:]
print train_set_data_vstacked_reshaped.shape
print train_set_output_vstacked_reshaped.shape
for train, test in kfold.split(train_set_data_vstacked_reshaped):
print "Model definition!"
model = Sequential()
#act = PReLU(init='normal', weights=None)
#model.add(Dense(output_dim=400,input_dim=400, init="normal", activation=K.tanh))
#act1 = PReLU(init='normal', weights=None)
#act2 = PReLU(init='normal', weights=None)
#model.add(Dense(output_dim=2050, input_dim=2050, init="he_normal",activation='tanh'))
#model.add(Dense(output_dim=13, input_dim=2050, init="he_normal",activation='relu'))
#model.add(Lambda(lambda x: numpy_unorm(x)))
#model.add(ELU(100))
model.add(Convolution1D(2050, 1, border_mode='same', input_dim=2050))
model.add(Convolution1D(13, 1, border_mode='same', input_dim=2050))
#model.add(Convolution1D(13, 1, border_mode='same', input_dim=2050))
#model.add(Dense(output_dim=13, input_dim=2050, init="he_normal",activation='relu'))
print "Compiling"
#rms_opt = keras.optimizers.RMSprop(lr=0.01, rho=0.9, epsilon=1e-08, decay=0.0)
model.compile(loss='mean_squared_error', optimizer="Adam")
print "Compile done! "
print '\n'
print "Train start"
reduce_lr=ReduceLROnPlateau(monitor='val_loss', factor=0.01, patience=3, verbose=1, mode='auto', epsilon=0.01, cooldown=0, min_lr=0.000000000000000001)
stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
log=csv_logger = CSVLogger('training_'+str(i)+'.csv')
hist_current = model.fit(train_set_data_vstacked_reshaped[train],
train_set_output_vstacked_reshaped[train],
shuffle=False,
validation_data=(train_set_data_vstacked_reshaped[test],train_set_output_vstacked_reshaped[test]),
validation_split=0.1,
nb_epoch=150,
verbose=1,
callbacks=[reduce_lr,log,stop])
hist.append(hist_current)
Related
I'm new in deep learning. In first step I create and train a model in python with keras and freezed by this code:
def export_model(MODEL_NAME, input_node_name, output_node_name):
tf.train.write_graph(K.get_session().graph_def, 'out', \
MODEL_NAME + '_graph.pbtxt')
tf.train.Saver().save(K.get_session(), 'out/' + MODEL_NAME + '.chkp')
freeze_graph.freeze_graph('out/' + MODEL_NAME + '_graph.pbtxt', None, \
False, 'out/' + MODEL_NAME + '.chkp', output_node_name, \
"save/restore_all", "save/Const:0", \
'out/frozen_' + MODEL_NAME + '.pb', True, "")
input_graph_def = tf.GraphDef()
with tf.gfile.Open('out/frozen_' + MODEL_NAME + '.pb', "rb") as f:
input_graph_def.ParseFromString(f.read())
output_graph_def = optimize_for_inference_lib.optimize_for_inference(
input_graph_def, [input_node_name], [output_node_name],
tf.float32.as_datatype_enum)
with tf.gfile.FastGFile('out/opt_' + MODEL_NAME + '.pb', "wb") as f:
f.write(output_graph_def.SerializeToString())
it's output :
checkpoint
Model.chkp.data-00000-of-00001
Model.chkp.index
Model.chkp.meta
Model_graph.pbtxt
frozen_Model.pb
opt_Model.pb
when I want to read the net in opencv c++ by readNetFromTensorflow :
String weights = "frozen_Model.pb";
String pbtxt = "Model_graph.pbtxt";
dnn::Net cvNet = cv::dnn::readNetFromTensorflow(weights, pbtxt);
This will make error :
OpenCV(4.0.0-pre) Error: Unspecified error (FAILED: ReadProtoFromBinaryFile(param_file, param). Failed to parse GraphDef file: frozen_Model.pb) in cv::dnn::ReadTFNetParamsFromBinaryFileOrDie, file D:\LIBS\OpenCV-4.00\modules\dnn\src\tensorflow\tf_io.cpp, line 44
and
OpenCV(4.0.0-pre) Error: Assertion failed (const_layers.insert(std::make_pair(name, li)).second) in cv::dnn::experimental_dnn_v4::`anonymous-namespace'::addConstNodes, file D:\LIBS\OpenCV-4.00\modules\dnn\src\tensorflow\tf_importer.cpp, line 555
How to fix this error?
Amin, may I ask you to try to save a graph in a testing mode:
K.backend.set_learning_phase(0) # <--- This setting makes all the following layers work in test mode
model = Sequential(name = MODEL_NAME)
model.add(Conv2D(filters = 128, kernel_size = (5, 5), activation = 'relu',name = 'FirstLayerConv2D_No1',input_shape = (Width, Height, image_channel)))
...
model.add(Dropout(0.25))
model.add(Dense(100, activation = 'softmax', name = 'endNode'))
# Create a graph definition (with no weights)
sess = K.backend.get_session()
sess.as_default()
tf.train.write_graph(sess.graph.as_graph_def(), "", 'graph_def.pb', as_text=False)
Then freeze your checkpoint files with a newly created graph_def.pb by a freeze_graph.py script (do not forget to use --input_binary flag).
Part of the code :
Create model, train and export_model
train_batch = gen.flow_from_directory(path + 'Train', target_size = (Width, Height), shuffle = False, color_mode = color_mode,
batch_size = batch_size_train, class_mode = 'categorical')
.
.
X_train, Y_train = next(train_batch)
.
.
X_train = X_train.reshape(X_train.shape).astype('float32')
.
.
model = Sequential(name = MODEL_NAME)
model.add(Conv2D(filters = 128, kernel_size = (5, 5), activation = 'relu',name = 'FirstLayerConv2D_No1',input_shape = (Width, Height, image_channel)))
model.add(Conv2D(filters = 128, kernel_size = (3, 3), activation = 'relu'))
model.add(MaxPool2D(pool_size = (2, 2)))
model.add(BatchNormalization())
.
.
.
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(200, activation = 'tanh'))
model.add(BatchNormalization())
model.add(Dropout(0.25))
model.add(Dense(100, activation = 'softmax', name = 'endNode'))
model.compile(loss = 'categorical_crossentropy',
optimizer = SGD(lr = 0.01, momentum = 0.9), metrics = ['accuracy'])
history = model.fit(X_train, Y_train, batch_size = batch_size_fit, epochs = epoch, shuffle = True,
verbose = 1, validation_split = .1, validation_data = (X_test, Y_test))
export_model(MODEL_NAME, "FirstLayerConv2D_No1/Relu", "endNode/Softmax")
When you're writing the graph in python you need to do the following steps:
with tf.Session(graph=tf.Graph()) as sess:
# 1. Load saved model
saved_model = tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], SAVED_MODEL_PATH)
# 2. Convert variables to constants
inference_graph_def = tf.graph_util.convert_variables_to_constants(sess, saved_model.graph_def, OUTPUT_NODE_NAMES)
# 3. Optimize for inference
optimized_graph_def = optimize_for_inference_lib.optimize_for_inference(inference_graph_def,
INPUT_NODE_NAMES,
OUTPUT_NODE_NAMES,
tf.float32.as_datatype_enum)
# 4. Save .pb file
tf.train.write_graph(optimized_graph_def, MODEL_DIR, 'model_name.pb', as_text=False)
# 5. Transform graph
transforms = [
'strip_unused_nodes(type=float, shape=\"1,128,128,3\")',
'remove_nodes(op=PlaceholderWithDefault)',
'remove_device',
'sort_by_execution_order'
]
transformed_graph_def = TransformGraph(optimized_graph_def,
INPUT_NODE_NAMES,
OUTPUT_NODE_NAMES,
transforms)
# 6. Remove constant nodes and attributes
for i in reversed(range(len(transformed_graph_def.node))):
if transformed_graph_def.node[i].op == "Const":
del transformed_graph_def.node[i]
for attr in ['T', 'data_format', 'Tshape', 'N', 'Tidx', 'Tdim',
'use_cudnn_on_gpu', 'Index', 'Tperm', 'is_training', 'Tpaddings']:
if attr in transformed_graph_def.node[i].attr:
del transformed_graph_def.node[i].attr[attr]
# 7. Save .pbtxt file
tf.train.write_graph(transformed_graph_def, MODEL_DIR, 'model_name.pbtxt', as_text=True)
Plus, if you have special nodes as Flatten, you need to remove and rename some nodes manually.
More info here.
I'm new to TF and ML.
Details about data: Features(x) - (70 x 70 x 70) tensor for each sample, y - a float for each sample.
TFRecords created with the following code:
def convert_to_tf_records():
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _float64_feature(value):
return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
tfrecords_filename = 'A-100-h2-h2o.tfrecords'
writer = tf.python_io.TFRecordWriter(tfrecords_filename)
# Get data from db for now.
db = connect('results-60-70.db')
data = db.select(selection='Ti')
i = 0
for row in data:
desc = np.array(json.loads(row.descriptor), dtype=np.float32)
print(desc.shape)
be = float(row.binding_energy) * 23 # Convert to Kcal/mol ?
desc = desc.flatten()
desc = desc.tostring()
example = tf.train.Example(features=tf.train.Features(feature={'voxel_grid': _bytes_feature(desc), 'binding_energy': _float64_feature(be)}))
writer.write(example.SerializeToString())
i += 1
if i >= 10:
break
Input function:
def my_input_function(fname, perform_shuffle=False, repeat_count=None):
def _parse_elements(example):
features = tf.parse_single_example(example, features={'voxel_grid': tf.FixedLenFeature([], tf.string), 'binding_energy': tf.FixedLenFeature([], tf.float32)})
vg = tf.decode_raw(features['voxel_grid'], tf.float32)
vg = tf.reshape(vg, [70, 70, 70])
vg = tf.convert_to_tensor(vg, dtype=tf.float32)
vg = {'voxel_grid': vg}
e = tf.cast(features['binding_energy'], tf.float32)
return vg, e
def input_function():
dataset = tf.data.TFRecordDataset(fname).map(_parse_elements)
dataset = dataset.repeat(repeat_count)
dataset = dataset.batch(5)
dataset = dataset.prefetch(1)
if perform_shuffle:
dataset.shuffle(20)
iterator = dataset.make_one_shot_iterator()
batch_features, batch_labels = iterator.get_next()
return batch_features, batch_labels
return input_function
Model function:
def my_model_function(features, labels, mode):
if mode == tf.estimator.ModeKeys.PREDICT:
tf.logging.info("my_model_fn: PREDICT, {}".format(mode))
elif mode == tf.estimator.ModeKeys.EVAL:
tf.logging.info("my_model_fn: EVAL, {}".format(mode))
elif mode == tf.estimator.ModeKeys.TRAIN:
tf.logging.info("my_model_fn: TRAIN, {}".format(mode))
feature_columns = [tf.feature_column.numeric_column('voxel_grid', shape=(70, 70, 70), dtype=tf.float32)]
# Create the layer of input
input_layer = tf.feature_column.input_layer(features, feature_columns)
input_layer = tf.reshape(input_layer, [-1, 70, 70, 70, 1])
# Convolution layers
conv1 = tf.layers.conv3d(inputs=input_layer, strides=(2, 2, 2), filters=32, kernel_size=(7, 7, 7))
conv2 = tf.layers.conv3d(inputs=conv1, strides=(2, 2, 2), filters=32, kernel_size=(7, 7, 7))
pool3 = tf.layers.max_pooling3d(inputs=conv2, pool_size=[2, 2, 2], strides=2)
flat = tf.layers.flatten(pool3)
dense1 = tf.layers.dense(inputs=flat, units=10, activation=tf.nn.relu)
dense2 = tf.layers.dense(inputs=dense1, units=10, activation=tf.nn.relu)
output = tf.layers.dense(inputs=dense2, units=1)
predictions = {'binding_energy': output}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Calculate loss
loss = tf.losses.mean_squared_error(labels=labels, predictions=predictions)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
# Add evaluation metrics
eval_metric_ops = {"mse": tf.metrics.mean_squared_error(labels=labels, predictions=predictions['binding_energy'])}
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
When calling model.train using
model = tf.estimator.Estimator(model_fn=my_model_function, model_dir='./model_dir')
model.train(input_fn=my_input_function('A-100-h2-h2o.tfrecords'), steps=100)
I get the following error.
TypeError: Failed to convert object of type to Tensor.
Found it!
changing
# Calculate loss
loss = tf.losses.mean_squared_error(labels=labels, predictions=predictions)
to
# Calculate loss
loss = tf.losses.mean_squared_error(labels=labels, predictions=predictions['binding_energy'])
solves the issue.
I'm trying to make speech recognition system with tensorflow.
Input data is an numpy array of size 50000 X 1.
Output data (mapping data) is an numpy array of size 400 X 1.
Input and mapping data is passed in batches of 2 in a list.
I've used this tutorial to design the neural network. Following is the code snippet:
For RNN:
input_data = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_DATA, sound_constants.MAX_COLUMN_SIZE_IN_DATA], name="train_input")
target = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT], name="train_output")
fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating one backward cell
bkwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating bidirectional RNN
val, _, _ = tf.nn.static_bidirectional_rnn(fwd_cell, bkwd_cell, tf.unstack(input_data), dtype=tf.float32)
For feeding data:
feed = {g['input_data'] : trb[0], g['target'] : trb[1], g['dropout'] : 0.6}
accuracy_, _ = sess.run([g['accuracy'], g['ts']], feed_dict=feed)
accuracy += accuracy_
When I ran the code, I got this error:
Traceback (most recent call last):
File "/home/wolborg/PycharmProjects/speech-to-text-rnn/src/rnn_train_1.py", line 205, in <module>
tr_losses, te_losses = train_network(g)
File "/home/wolborg/PycharmProjects/speech-to-text-rnn/src/rnn_train_1.py", line 177, in train_network
accuracy_, _ = sess.run([g['accuracy'], g['ts']], feed_dict=feed)
File "/home/wolborg/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 895, in run
run_metadata_ptr)
File "/home/wolborg/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1102, in _run
raise ValueError('Tensor %s may not be fed.' % subfeed_t)
ValueError: Tensor Tensor("Const:0", shape=(), dtype=float32) may not be fed.
Process finished with exit code 1
Earlier, I was facing this issue with tf.sparse_placeholder, then after some browsing, I changed input type to tf.placeholder and made related changes. Now I'm clueless on where I'm making the error.
Please suggest something as how should I feed data.
Entire code:
import tensorflow as tf
# for taking MFCC and label input
import numpy as np
import rnn_input_data_1
import sound_constants
# input constants
# Training Parameters
num_input = 10 # mfcc data input
training_data_size = 8 # determines number of files in training and testing module
testing_data_size = num_input - training_data_size
# Network Parameters
learning_rate = 0.0001 # for large training set, it can be set 0.001
num_hidden = 200 # number of hidden layers
num_classes = 28 # total alphabet classes (a-z) + extra symbols (', ' ')
epoch = 1 # number of iterations
batch_size = 2 # number of batches
mfcc_coeffs, text_data = rnn_input_data_1.mfcc_and_text_encoding()
class DataGenerator:
def __init__(self, data_size):
self.ptr = 0
self.epochs = 0
self.data_size = data_size
def next_batch(self):
self.ptr += batch_size
if self.ptr > self.data_size:
self.epochs += 1
self.ptr = 0
return mfcc_coeffs[self.ptr-batch_size : self.ptr], text_data[self.ptr-batch_size : self.ptr]
def reset_graph():
if 'sess' in globals() and sess:
sess.close()
tf.reset_default_graph()
def struct_network():
print ('Inside struct network !!')
reset_graph()
input_data = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_DATA, sound_constants.MAX_COLUMN_SIZE_IN_DATA], name="train_input")
target = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT], name="train_output")
keep_prob = tf.constant(1.0)
fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating one backward cell
bkwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating bidirectional RNN
val, _, _ = tf.nn.static_bidirectional_rnn(fwd_cell, bkwd_cell, tf.unstack(input_data), dtype=tf.float32)
# adding dropouts
val = tf.nn.dropout(val, keep_prob)
val = tf.transpose(val, [1, 0, 2])
last = tf.gather(val, int(val.get_shape()[0]) - 1)
# creating bidirectional RNN
print ('BiRNN created !!')
print ('Last Size: ', last.get_shape())
weight = tf.Variable(tf.truncated_normal([num_hidden * 2, sound_constants.MAX_ROW_SIZE_IN_TXT]))
bias = tf.Variable(tf.constant(0.1, shape=[sound_constants.MAX_ROW_SIZE_IN_TXT]))
# mapping to 28 output classes
logits = tf.matmul(last, weight) + bias
prediction = tf.nn.softmax(logits)
prediction = tf.reshape(prediction, shape = [batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT])
# getting probability distribution
mat1 = tf.cast(tf.argmax(prediction,1),tf.float32)
correct = tf.equal(prediction, target)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
logits = tf.reshape(logits, shape=[batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT])
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=target))
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)
# returning components as dictionary elements
return {'input_data' : input_data,
'target' : target,
'dropout': keep_prob,
'loss': loss,
'ts': train_step,
'preds': prediction,
'accuracy': accuracy
}
def train_network(graph):
# initialize tensorflow session and all variables
# tf_gpu_config = tf.ConfigProto(allow_soft_placement = True, log_device_placement = True)
# tf_gpu_config.gpu_options.allow_growth = True
# with tf.Session(config = tf_gpu_config) as sess:
with tf.Session() as sess:
train_instance = DataGenerator(training_data_size)
test_instance = DataGenerator(testing_data_size)
print ('Training data size: ', train_instance.data_size)
print ('Testing data size: ', test_instance.data_size)
sess.run(tf.global_variables_initializer())
print ('Starting session...')
step, accuracy = 0, 0
tr_losses, te_losses = [], []
current_epoch = 0
while current_epoch < epoch:
step += 1
trb = train_instance.next_batch()
feed = {g['input_data'] : trb[0], g['target'] : trb[1], g['dropout'] : 0.6}
accuracy_, _ = sess.run([g['accuracy'], g['ts']], feed_dict=feed)
accuracy += accuracy_
if train_instance.epochs > current_epoch:
current_epoch += 1
tr_losses.append(accuracy / step)
step, accuracy = 0, 0
#eval test set
te_epoch = test_instance.epochs
while test_instance.epochs == te_epoch:
step += 1
print ('Testing round ', step)
trc = test_instance.next_batch()
feed = {g['input_data']: trc[0], g['target']: trc[1]}
accuracy_ = sess.run([g['accuracy']], feed_dict=feed)[0]
accuracy += accuracy_
te_losses.append(accuracy / step)
step, accuracy = 0,0
print("Accuracy after epoch", current_epoch, " - tr:", tr_losses[-1], "- te:", te_losses[-1])
return tr_losses, te_losses
g = struct_network()
tr_losses, te_losses = train_network(g)
You defined keep_prob as a tf.constant, but then trying to feed the value into it. Replace keep_prob = tf.constant(1.0) with keep_prob = tf.placeholder(tf.float32,[]) or keep_prob = tf.placeholder_with_default(1.0,[])
I am struggling to understand why i am getting such a high loss/val_loss rate on my training. I am training a regression network. I've normalized the input data to range between -1 to 1, and left the output data unaltered, its range is approx. between -100 to 100.
I chose to normalize the input such that i could use tanh, as activation function as it outputs within this range.
The neural network consist of 3 layers.
print "Model definition!"
model = Sequential()
#act = PReLU(init='normal', weights=None)
model.add(Dense(output_dim=400,input_dim=400, init="normal",activation=K.tanh))
#act1 = PReLU(init='normal', weights=None)
model.add(Dense(output_dim=400,input_dim=400, init="normal",activation=K.tanh))
#act2 = PReLU(init='normal', weights=None)
#model.add(Dense(output_dim=400, input_dim=400, init="normal",activation=K.tanh))
act4=ELU(100)
model.add(Dense(output_dim=13, input_dim=400, init="normal",activation=act4))
The mapping between the input and output consist of mapping audio samples into MFCC features. The samples are the ones i've normalized to the described range.
why am i getting these results?
what could/should i improve/change ?
cv code running:
#Each frame length = 118
#Each feature length = 13
############################### Training setup ##################################
#Define 10 folds:
seed = 7
np.random.seed(seed)
kfold = KFold(n_splits=10, shuffle=False, random_state=None)
print "Splits"
cvscores_acc = []
cvscores_loss = []
hist = []
i = 0
for train, test in kfold.split(train_set_data_vstacked_normalized):
print "Model definition!"
model = Sequential()
#act = PReLU(init='normal', weights=None)
model.add(Dense(output_dim=400,input_dim=400, init="normal",activation=K.tanh))
#act1 = PReLU(init='normal', weights=None)
model.add(Dense(output_dim=400,input_dim=400, init="normal",activation=K.tanh))
#act2 = PReLU(init='normal', weights=None)
#model.add(Dense(output_dim=400, input_dim=400, init="normal",activation=K.tanh))
act4=ELU(100)
model.add(Dense(output_dim=13, input_dim=400, init="normal",activation=act4))
print "Compiling"
model.compile(loss='mean_squared_error', optimizer='RMSprop')
print "Compile done! "
print '\n'
print "Train start"
reduce_lr=ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0.00000001)
log=csv_logger = CSVLogger('training_'+str(i)+'.csv')
hist_current = model.fit(train_set_data_vstacked_normalized[test],train_set_output_vstacked[test], shuffle=False,validation_split=0.1 , nb_epoch=1000,verbose=1,callbacks=[reduce_lr,log])
hist.append(hist_current)
loss, accuracy = model.evaluate(x=train_set_data_vstacked_normalized[train],y=train_set_output_vstacked[train],verbose=1)
print
print('loss: ', loss)
print('accuracy: ', accuracy)
print()
print model.summary()
print "New Model:"
cvscores_acc.append(accuracy)
cvscores_loss.append(loss)
print "Model stored"
model.save("Model"+str(i))
i=i+1
print("%.2f%% (+/- %.2f%%)" % (numpy.mean(cvscores_acc), numpy.std(cvscores_acc)))
print("%.2f%% (+/- %.2f%%)" % (numpy.mean(cvscores_loss), numpy.std(cvscores_loss)))
https://www.dropbox.com/sh/zeaa8ouwq3oo8c3/AAAgRNJwRL_TGwFWkBD6KFV0a?dl=0
My data: data.csv
I tried to draw a contour plot using Python.
import numpy as np
import matplotlib.pyplot as plt
import scipy.interpolate
import time
import datetime
f = open("data.csv","r")
title = f.readline()
lines = f.readlines()
print len(lines)
xlist = []
ylist = []
zlist = []
for line in lines:
titlecells = title.split(',')
linecells = line.split(',')
print len(titlecells)
for i in range(1,len(titlecells)):
items = titlecells[i].split('/')
items = map(int, items)
dt = datetime.datetime(items[0], items[1], items[2])
sdt = datetime.datetime(1900,1,1)
delta = int(str(dt - sdt).split(' ')[0]) / 10
xlist.append(delta)
ylist.append(linecells[0].replace('0~','').replace('m',''))
try:
zlist.append(float(linecells[i]))
except ValueError:
zlist.append(0)
print len(xlist)
print len(ylist)
print len(zlist)
ylist = map(int, ylist)
x = np.array(xlist)
y = np.array(ylist)
z = np.array(zlist)
print z
# Generate data: for N=1e6, the triangulation hogs 1 GB of memory
'''N = 1000000
x, y = 10 * np.random.random((2, N))
rho = np.sin(3*x) + np.cos(7*y)**3
print x.shape,y.shape'''
# Set up a regular grid of interpolation points
xi, yi = np.linspace(x.min(), x.max(), 300), np.linspace(y.min(), y.max(), 300)
xi, yi = np.meshgrid(xi, yi)
print xi,yi
# Interpolate; there's also method='cubic' for 2-D data such as here
zi = scipy.interpolate.griddata((x, y), z, (xi, yi), method='linear')
plt.imshow(zi, vmin=z.min(), vmax=z.max(), origin='lower',
extent=[x.min(), x.max(), y.min(), y.max()])
plt.colorbar()
plt.show()
Output obtained with Matlab:
Input data: TKJS_20150903.xlsx
Matlab Code
clear all; close all;
% read data
list_d2 = dir('C:\\MATLAB_CODE\\Monitoring_well\\TS_Analysis\\MW\\data\\*.xlsx');
for k = 1:length(list_d2)
station = list_d2(k).name(1:13);
filename = sprintf('C:\\MATLAB_CODE\\Monitoring_well\\TS_Analysis\\MW\\data\\%s.xlsx', station);
[MW_data, data_tex, alldata] = xlsread(filename,5);
MW_data = MW_data(2:end,:);
date = datenum(cellfun(#num2str,alldata(3:end,1),'UniformOutput', false),'yyyy/mm/dd');
depth = cell2mat(alldata(2,2:end))';
% MW_data=MW_data';
% compute gap between every rings
for i = 1:length(MW_data(:,1))
for j = 1:length(MW_data(1,:))-1
disp(i,j) = MW_data(i,j)-MW_data(i,j+1);
end
end
vel=disp';
% dlmwrite('displacement.txt', disp, '\t');
for i = 2:length(depth)-1
thick(i) = depth(i+1)-depth(i);
end
thick=thick(1:end);
thick1=repmat(thick,length(disp(:,1)),1);
% dlmwrite('thickness.txt', thick, '\t');
% % check data
% figure(1);
% plot(MW_data,depth*(-1));
% set well depth
yg1 = 0; yg2 = 300;
% make grid
[xgrid,ygrid] = meshgrid(date(1):30:date(end),yg1:1:yg2);
xint=repmat(date',length(thick),1);
yint=repmat(depth(2:end),1,length(date));
xint1 = reshape(xint, numel(xint), 1);
yint1 = reshape(yint, numel(yint), 1);
zint1 = reshape(vel, numel(vel), 1);
% [Xwdp,Ywdp,Zwdp] = griddata(xint1,yint1,zint1,xgrid,ygrid);
[Xwdp,Ywdp,Zwdp] = surfergriddata(xint1, yint1, zint1, xgrid, ygrid, 'kriging');
% save('cresult.mat');
h1 = figure;
set(h1,'paperorientation', 'portrait', 'color', [1 1 1], 'papertype','A4');
set(gcf,'Units','Centimeter','Position', [5 5 18 20]);
set(gcf,'PaperPositionMode','auto');
set(gca, 'FontSize', 12, 'FontWeight','bold');
set(gca, 'FontName', 'Arial')
% box on;
% pcolor(xgrid, ygrid, Zwdp);
surf(xgrid,ygrid,Zwdp);
shading interp;
view(0,90);
colorbar;
z1=round(min(min(vel)));
z2=round(max(max(vel)));
caxis([z1 z2]);
% caxis([-0.5 0.5]);
% caxis([-1.2 1.2]);
% caxis([-1 2]);
% caxis([-1 8]);
% x1=floor(min(min(date)));
% x2=round(max(max(date)));
axis([ min(min(date)), max(max(date)), 0, 300]);
set(gca,'fontsize',5,'xticklabelmode','manual','yticklabelmode','manual');
pa = get(gca,'xlim');
pb = get(gca,'ylim');
pa1 = (pa(1):pa(2));
pa2 = (pb(1):pb(2));
indx = find(mod(pa1-pa1(1),60)==0);
indy = find(mod(pa2,50)==0);
set(gca,'xtick',pa1(indx),'xticklabel',[]);
xrtime=get(gca,'xtick');
xstime=datestr(xrtime,'yyyy/mm');
for n=1:length(xrtime)
text(xrtime(n),308,xstime(n,:),'HorizontalAlignment','center','fontsize',8,'fontweight','bold')
end
set(gca,'ytick',pa2(indy),'yticklabel',pa2(indy),'fontsize',8,'fontweight','bold');
set(gca,'YDir','reverse','tickdir','out'); hold on
% gridnumy = zint1(:,1);
%
% for j = 1:length(gridnumy)
% line(xgrid(1,:)',repmat(gridnumy(j),size(xgrid,2),1),'color','black','linestyle','-');
% end
well=list_d2(k).name(1:4)
title({well 'Monitoring Well'}, 'FontSize', 12, 'FontWeight','bold');
xlabel('Date', 'FontSize', 12, 'FontWeight','bold');
ylabel('Depth (m)', 'Fontsize', 12, 'FontWeight','bold');
% clabel('Compression (cm)', 'Fontsize', 12, 'FontWeight','bold');
list_p4=sprintf('C:\\MATLAB_CODE\\Monitoring_well\\TS_Analysis\\MW\\%s', well);
print('-dpng','-r300', list_p4);
clear MW_data data_tex xint xint1 yint yint1 zint1 thick thick1 vel disp
end
I want to get similar Output using Python.
But, I don't know how to improve my code.