I have a problem with my global variables in my code. In SCRIPT1.py I use many variables from a little document config.py which only contains variables which I also need in other modules of my code. But when running my SCRIPT1.py I get an error (ERROR). I have no idea why it doesn't work with config.(name of variable)... I found this solution to have your variables in all of your modules on stack overflow with a lot of good votes. What am I doing wrong?
First my code contained config.costSurfaceA in stead of costSurfaceArray (for ex in 'def createPath') but when running it with this variable, it gave me a syntax error because of the dot in 'config.costSurfaceA'. I replaced it all by 'costSurfaceArray' and did this in the if statement 'config.costSurfaceA = costSurfaceArray' just to get it as a variable. But I have the feeling this is all to much work for nothing..
Thanks in avance for helping me! I know it is a lot of code but I think it's all important for understanding..
SCRIPT1.py
from osgeo import gdal, osr
from skimage.graph import route_through_array
import numpy as np
import Save_Array_To_Excel_01
import config
def ask_costsurfacepath_path():
config.costsurfacepath = input('please enter the system path where to find the cost-surface-IMG file (ex: /Users/PeterVanvoorden/Documents/GroepT/Thesis/Branched_Testfile.img): ')
def ask_outputpath_path():
config.outputpath = input('please enter the system path where to save the outputpath IMG file (ex: /Users/PeterVanvoorden/Documents/GroepT/Thesis/Branched_Testfile.img): ')
def raster2array(rasterfn):
print 'raster2array'
raster = gdal.Open(rasterfn)
band = raster.GetRasterBand(1)
array = band.ReadAsArray()
return array
def coord2pixelOffset(rasterfn,x,y):
print 'coord2pixelOffset'
raster = gdal.Open(rasterfn)
geotransform = raster.GetGeoTransform()
originX = geotransform[0] # East/West location of Upper Left corner
originY = geotransform[3] # North/South location of Upper Left corner
pixelWidth = geotransform[1] # X pixel size
pixelHeight = geotransform[5] # Y pixel size
xOffset = int((x - originX)/pixelWidth)
yOffset = int((y - originY)/pixelHeight)
return xOffset,yOffset
def createPath(CostSurfacefn,costSurfaceArray,startCoord,stopCoord):
print 'creatpath'
# coordinates to array index
startCoordX = startCoord[0]
startCoordY = startCoord[1]
startIndexX,startIndexY = coord2pixelOffset(CostSurfacefn,startCoordX,startCoordY)
stopCoordX = stopCoord[0]
stopCoordY = stopCoord[1]
stopIndexX,stopIndexY = coord2pixelOffset(CostSurfacefn,stopCoordX,stopCoordY)
# create path
indices, weight = route_through_array(costSurfaceArray, (startIndexY,startIndexX), (stopIndexY,stopIndexX),geometric=True,fully_connected=True)
indices = np.array(indices).T
path = np.zeros_like(costSurfaceArray)
path[indices[0], indices[1]] = 1
return path
def array2raster(newRasterfn,rasterfn,array):
print 'array2raster'
raster = gdal.Open(rasterfn)
geotransform = raster.GetGeoTransform()
originX = geotransform[0] # East/West location of Upper Left corner
originY = geotransform[3] # North/South location of Upper Left corner
pixelWidth = geotransform[1] # X pixel size
pixelHeight = geotransform[5] # Y pixel size
cols = array.shape[1]
rows = array.shape[0]
driver = gdal.GetDriverByName('GTiff')
outRaster = driver.Create(newRasterfn, cols, rows, gdal.GDT_Byte)
outRaster.SetGeoTransform((originX, pixelWidth, 0, originY, 0, pixelHeight))
outband = outRaster.GetRasterBand(1)
outband.WriteArray(array)
outRasterSRS = osr.SpatialReference()
outRasterSRS.ImportFromWkt(raster.GetProjectionRef())
outRaster.SetProjection(outRasterSRS.ExportToWkt())
outband.FlushCache()
def main(CostSurfacefn,outputPathfn,startCoord,stopCoord):
print 'main'
costSurfaceArray = raster2array(CostSurfacefn) # creates array from cost surface raster
config.costSurfaceA = costSurfaceArray
config.pathArray = createPath(CostSurfacefn,costSurfaceArray,startCoord,stopCoord) # creates path array
Save_Array_To_Excel_01.Save_Array(config.pathArray) # Save Array to csv file
array2raster(outputPathfn,CostSurfacefn,config.pathArray) # converts path array to raster
if __name__ == "__main__":
ask_costsurfacepath_path()
ask_outputpath_path()
CostSurfacefn = config.costsurfacepath
print config.costsurfacepath
startCoord = (config.startX,config.startY)
stopCoord = (config.stopX,config.stopY)
outputPathfn = config.outputpath
main(CostSurfacefn,outputPathfn,startCoord,stopCoord)
config.py
# Configuration file with all global variables
# number of properties
number = None
# different permutations of properties
permutations = list()
# properties array containing:
# * first column = ID first property [0]
# * second column = ID second property [1]
# * third column = distance between two properties [2]
# * forth column = estimated cost [3]
properties_array = None
# lowest price until now
lowest_price = 10**10000
# path with this lowest price
lowest_path = None
# current price (needs to be compared with lowest price)
current_price = 0
# current path (needs to be compared with lowest path)
current_path = [1]
# path to place where to save properties list
plist_path = None
# Array of the path
pathArray = None
# Array of the map
costSurfaceA = None
# current start X coordinate
startX = 0
# current start Y coordinate
startY = 0
# current stop X coordinate
stopX = 0
# current stop Y coordinate
stopY = 0
# path to costsurface IMG file
costsurfacepath = 0
# path to output path from Least cost path analysis
outputpath = 0
ERROR
please enter the system path where to put the file as a STRING (ex: /Users/PeterVanvoorden/Documents/GroepT/Thesis/Branched_Testfile.csv): '/User/PeterVanvoorden/Desktop/Shell.csv'
You entered: /User/PeterVanvoorden/Desktop/Shell.csv
please enter the system path where to find the cost-surface-IMG file (ex: /Users/PeterVanvoorden/Documents/GroepT/Thesis/Branched_Testfile.img): '/User/PeterVanvoorden/Desktop/clipsmall.img'
please enter the system path where to save the outputpath IMG file (ex: /Users/PeterVanvoorden/Documents/GroepT/Thesis/Branched_Testfile.img): '/User/PeterVanvoorden/Desktop/Shellimg.img'
/User/PeterVanvoorden/Desktop/clipsmall.img
main
raster2array
Traceback (most recent call last):
File "/Users/PeterVanvoorden/Documents/GroepT/Thesis/f_python_standalone/python_files/Working_Files/Least_cost_path_analysis_01_outputArray.py", line 97, in <module>
main(CostSurfacefn,outputPathfn,startCoord,stopCoord)
File "/Users/PeterVanvoorden/Documents/GroepT/Thesis/f_python_standalone/python_files/Working_Files/Least_cost_path_analysis_01_outputArray.py", line 76, in main
costSurfaceArray = raster2array(CostSurfacefn) # creates array from cost surface raster
File "/Users/PeterVanvoorden/Documents/GroepT/Thesis/f_python_standalone/python_files/Working_Files/Least_cost_path_analysis_01_outputArray.py", line 17, in raster2array
band = raster.GetRasterBand(1)
AttributeError: 'NoneType' object has no attribute 'GetRasterBand'
You can pass around like this.
def ask_costsurfacepath_path():
costsurfacepath = input('please enter ...')
return costsurfacepath
... in __name__ == '__main__'
CostSurfacefn = ask_costsurfacepath_path()
...
Related
i was training a model, but it say that it is not enough values to unpack at the decoder. Here is the training step
def train(loader, encoder, decoder, optimizer, criterion, epoch):
loss_tracker, acc_tracker = AvgMeter(), AvgMeter()
encoder.train()
decoder.train()
pbar = tqdm.tqdm(enumerate(loader), total=len(loader))
for i, (images, target_sequences, sequence_lengths) in pbar:
images = images.to(device)
target_sequences = target_sequences.to(device)
sequence_lengths = sequence_lengths.to(device)
# Forward prop.
logits, alphas, sorted_target_sequences, sorted_decode_lengths, sorted_indices = decoder(images, target_sequences, sequence_lengths)
# Since we decoded starting with <sos>, the targets are all words after <sos>, up to <eos>
sorted_target_sequences = sorted_target_sequences[:, 1:]
# Remove paddings
logits = pack_padded_sequence(logits, sorted_decode_lengths).data
sorted_target_sequences = pack_padded_sequence(sorted_target_sequences, sorted_decode_lengths, batch_first=True).data
# Calculate loss
loss = criterion(logits, sorted_target_sequences)
# Add doubly stochastic attention regularization
loss += alpha_c * ((1. - alphas.sum(dim=1)) ** 2).mean()
# Back prop.
optimizer.zero_grad()
loss.backward()
# Clip gradients
if grad_clip is not None:
clip_gradient(optimizer, grad_clip)
# Update weights
optimizer.step()
# Track metrics
loss_tracker.update(loss.item(), sum(sorted_decode_lengths))
acc_tracker.update(accuracy(logits, sorted_target_sequences, 5), sum(sorted_decode_lengths))
# Update progressbar description
pbar.set_description(f'Epoch: {epoch + 1:03d} - train_loss: {loss_tracker.avg:.3f} - train_acc: {acc_tracker.avg:.3f}%')
return loss_tracker.avg, acc_tracker.avg
# , train_bleu1, train_bleu2, train_bleu3, train_bleu4
I was trying to solve the error by change the images, target_seq, and length into for i, (images, (target_sequences, sequence_lengths)) in pbar: then the error continue to it
<ipython-input-89-edf18c02b224> in process_training(encoder, decoder, optimizer, criterion, train_loader, valid_loader, field, alpha_c, start_epoch, n_epochs, grad_clip, model_name, last_improv)
28 # Train step
29 train_loss, train_acc = train(loader=train_loader, encoder=encoder, decoder=decoder, optimizer=optimizer, criterion=criterion,
---> 30 epoch=epoch)
31 # Validation step
32 val_loss, val_acc, bleu1, bleu2, bleu3, bleu4 = validate(encoder=encoder,decoder=decoder, criterion=criterion, loader=valid_loader,
<ipython-input-102-96da9ec6e1d4> in train(loader, encoder, decoder, optimizer, criterion, epoch)
9 sequence_lengths = sequence_lengths.to(device)
10 # Forward prop.
---> 11 logits, alphas, sorted_target_sequences, sorted_decode_lengths, sorted_indices = decoder(images, target_sequences, sequence_lengths)
12 # Since we decoded starting with <sos>, the targets are all words after <sos>, up to <eos>
13 sorted_target_sequences = sorted_target_sequences[:, 1:]
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1129 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130 return forward_call(*input, **kwargs)
1131 # Do not call functions when jit is used
1132 full_backward_hooks, non_full_backward_hooks = [], []
<ipython-input-97-da50cbd97d62> in forward(self, encoder_out, encoded_captions, caption_lengths)
88 # caption_lengths, sort_ind = caption_lengths.squeeze(1).sort(dim=0, descending=True)
89
---> 90 caption_lengths, sort_ind = caption_lengths.squeeze(1). sort(dim=0, descending=True)
91
92
IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)
Here is the code i use in the decoder to forward the input. It's error in the line of caption_lengths, sort_ind = caption_lengths.squeeze(1).sort(dim=0, descending=True) to sort caption data
def forward(self, encoder_out, encoded_captions, caption_lengths):
"""
Forward propagation.
:param encoder_out: encoded images, a tensor of dimension (batch_size, enc_image_size, enc_image_size, encoder_dim)
:param encoded_captions: encoded captions, a tensor of dimension (batch_size, max_caption_length)
:param caption_lengths: caption lengths, a tensor of dimension (batch_size, 1)
:return: scores for vocabulary, sorted encoded captions, decode lengths, weights, sort indices
"""
batch_size = encoder_out.size(0)
encoder_dim = encoder_out.size(-1)
vocab_size = self.vocab_size
# Flatten image
encoder_out = encoder_out.view(batch_size, -1, encoder_dim) # (batch_size, num_pixels, encoder_dim)
num_pixels = encoder_out.size(1)
# Sort input data by decreasing lengths; why? apparent below
caption_lengths, sort_ind = caption_lengths.squeeze(1).sort(dim=0, descending=True)
encoder_out = encoder_out[sort_ind]
encoded_captions = encoded_captions[sort_ind]
# Embedding
embeddings = self.embedding(encoded_captions) # (batch_size, max_caption_length, embed_dim)
# Initialize LSTM state
h, c = self.init_hidden_state(encoder_out) # (batch_size, decoder_dim)
# We won't decode at the <end> position, since we've finished generating as soon as we generate <end>
# So, decoding lengths are actual lengths - 1
decode_lengths = (caption_lengths - 1).tolist()
# Create tensors to hold word predicion scores and alphas
predictions = torch.zeros(batch_size, max(decode_lengths), vocab_size).to(device)
alphas = torch.zeros(batch_size, max(decode_lengths), num_pixels).to(device)
# At each time-step, decode by
# attention-weighing the encoder's output based on the decoder's previous hidden state output
# then generate a new word in the decoder with the previous word and the attention weighted encoding
for t in range(max(decode_lengths)):
batch_size_t = sum([l > t for l in decode_lengths])
attention_weighted_encoding, alpha = self.attention(encoder_out[:batch_size_t],
h[:batch_size_t])
gate = self.sigmoid(self.f_beta(h[:batch_size_t])) # gating scalar, (batch_size_t, encoder_dim)
attention_weighted_encoding = gate * attention_weighted_encoding
h, c = self.decode_step(
torch.cat([embeddings[:batch_size_t, t, :], attention_weighted_encoding], dim=1),
(h[:batch_size_t], c[:batch_size_t])) # (batch_size_t, decoder_dim)
preds = self.fc(self.dropout(h)) # (batch_size_t, vocab_size)
predictions[:batch_size_t, t, :] = preds
alphas[:batch_size_t, t, :] = alpha
return predictions, encoded_captions, decode_lengths, alphas,
I'm pretty new to deep learning, but I want to add Tensorboard to the following code to track loss, accuracy, average precision and so on.
Sample code from the TorchVision -2.3 Object Detection Finetuning Tutorial
http://pytorch.org/tutorials/intermediate/torchvision_tutorial.html
import os
import numpy as np
import torch
from PIL import Image
import sys
import torchvision
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor
from engine import train_one_epoch, evaluate
import utils
import transforms as T
#from torch.utils.tensorboard import SummaryWriter
#writer = SummaryWriter()
class PennFudanDataset(object):
def __init__(self, root, transforms):
self.root = root
self.transforms = transforms
# load all image files, sorting them to
# ensure that they are aligned
self.imgs = list(sorted(os.listdir(os.path.join(root, "PNGImages"))))
self.masks = list(sorted(os.listdir(os.path.join(root, "PedMasks"))))
def __getitem__(self, idx):
# load images and masks
img_path = os.path.join(self.root, "PNGImages", self.imgs[idx])
mask_path = os.path.join(self.root, "PedMasks", self.masks[idx])
img = Image.open(img_path).convert("RGB")
# note that we haven't converted the mask to RGB,
# because each color corresponds to a different instance
# with 0 being background
mask = Image.open(mask_path)
mask = np.array(mask)
# instances are encoded as different colors
obj_ids = np.unique(mask)
# first id is the background, so remove it
obj_ids = obj_ids[1:]
# split the color-encoded mask into a set
# of binary masks
masks = mask == obj_ids[:, None, None]
# get bounding box coordinates for each mask
num_objs = len(obj_ids)
boxes = []
for i in range(num_objs):
pos = np.where(masks[i])
xmin = np.min(pos[1])
xmax = np.max(pos[1])
ymin = np.min(pos[0])
ymax = np.max(pos[0])
boxes.append([xmin, ymin, xmax, ymax])
boxes = torch.as_tensor(boxes, dtype=torch.float32)
# there is only one class
labels = torch.ones((num_objs,), dtype=torch.int64)
masks = torch.as_tensor(masks, dtype=torch.uint8)
image_id = torch.tensor([idx])
area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
# suppose all instances are not crowd
iscrowd = torch.zeros((num_objs,), dtype=torch.int64)
target = {}
target["boxes"] = boxes
target["labels"] = labels
target["masks"] = masks
target["image_id"] = image_id
target["area"] = area
target["iscrowd"] = iscrowd
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target
def __len__(self):
return len(self.imgs)
def get_model_instance_segmentation(num_classes):
# load an instance segmentation model pre-trained pre-trained on COCO
model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True)
# get number of input features for the classifier
in_features = model.roi_heads.box_predictor.cls_score.in_features
# replace the pre-trained head with a new one
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
# now get the number of input features for the mask classifier
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
# and replace the mask predictor with a new one
model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,
hidden_layer,
num_classes)
return model
def get_transform(train):
transforms = []
transforms.append(T.ToTensor())
if train:
transforms.append(T.RandomHorizontalFlip(0.5))
return T.Compose(transforms)
def main():
# train on the GPU or on the CPU, if a GPU is not available
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
# our dataset has two classes only - background and person
num_classes = 2
# use our dataset and defined transformations
dataset = PennFudanDataset('PennFudanPed', get_transform(train=True))
dataset_test = PennFudanDataset('PennFudanPed', get_transform(train=False))
# split the dataset in train and test set
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])
# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(
dataset, batch_size=4, shuffle=True, num_workers=4,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(
dataset_test, batch_size=2, shuffle=False, num_workers=4,
collate_fn=utils.collate_fn)
# get the model using our helper function
model = get_model_instance_segmentation(num_classes)
# move model to the right device
model.to(device)
# construct an optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.005,
momentum=0.9, weight_decay=0.0005)
# and a learning rate scheduler
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=0.1)
# let's train it for 10 epochs
num_epochs = 10
for epoch in range(num_epochs):
# train for one epoch, printing every 10 iterations
train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq=10)
# update the learning rate
lr_scheduler.step()
# evaluate on the test dataset
evaluate(model, data_loader_test, device=device)
print("That's it!")
if __name__ == "__main__":
main()
I have a list of images to make a collage named in a certain pattern.
Ex.
yahoo_jp.png
yahoo_us.png
yahoo_uk.png
yahoo_cn.png
All files are in the same directory. Currently, I can only send a command to make a collage out of all images in a folder but what I want to do is to be able to send a certain keyword from the shell command and look up a list of files in the folder with the keywords matching and then making a collage.
Ex. shell command
make_collage.py -o my_collage.png -w 540 -i 840 new argument --> -a "_us"
- when this command is run, it will only make a collage with files containing keyword "_us" only. So the output would be a collage containing only the "_us" images.
import argparse
import os
import random
from PIL import Image
def make_collage(images, filename, width, init_height):
"""
Make a collage image with a width equal to `width` from `images` and save to `filename`.
"""
if not images:
print('No images for collage found!')
return False
margin_size = 2
# run until a suitable arrangement of images is found
while True:
# copy images to images_list
images_list = images[:]
coefs_lines = []
images_line = []
x = 0
while images_list:
# get first image and resize to `init_height`
img_path = images_list.pop(0)
img = Image.open(img_path)
img.thumbnail((width, init_height))
# when `x` will go beyond the `width`, start the next line
if x > width:
coefs_lines.append((float(x) / width, images_line))
images_line = []
x = 0
x += img.size[0] + margin_size
images_line.append(img_path)
# finally add the last line with images
coefs_lines.append((float(x) / width, images_line))
# compact the lines, by reducing the `init_height`, if any with one or less images
if len(coefs_lines) <= 1:
break
if any(map(lambda c: len(c[1]) <= 1, coefs_lines)):
# reduce `init_height`
init_height -= 10
else:
break
# get output height
out_height = 0
for coef, imgs_line in coefs_lines:
if imgs_line:
out_height += int(init_height / coef) + margin_size
if not out_height:
print('Height of collage could not be 0!')
return False
collage_image = Image.new('RGB', (width, int(out_height)), (35, 35, 35))
# put images to the collage
y = 0
for coef, imgs_line in coefs_lines:
if imgs_line:
x = 0
for img_path in imgs_line:
img = Image.open(img_path)
# if need to enlarge an image - use `resize`, otherwise use `thumbnail`, it's faster
k = (init_height / coef) / img.size[1]
if k > 1:
img = img.resize((int(img.size[0] * k), int(img.size[1] * k)), Image.ANTIALIAS)
else:
img.thumbnail((int(width / coef), int(init_height / coef)), Image.ANTIALIAS)
if collage_image:
collage_image.paste(img, (int(x), int(y)))
x += img.size[0] + margin_size
y += int(init_height / coef) + margin_size
collage_image.save(filename)
return True
def main():
# prepare argument parser
parse = argparse.ArgumentParser(description='Photo collage maker')
parse.add_argument('-f', '--folder', dest='folder', help='folder with images (*.jpg, *.jpeg, *.png)', default='.')
parse.add_argument('-o', '--output', dest='output', help='output collage image filename', default='collage.png')
parse.add_argument('-w', '--width', dest='width', type=int, help='resulting collage image width')
parse.add_argument('-i', '--init_height', dest='init_height', type=int, help='initial height for resize the images')
parse.add_argument('-s', '--shuffle', action='store_true', dest='shuffle', help='enable images shuffle')
args = parse.parse_args()
if not args.width or not args.init_height:
parse.print_help()
exit(1)
# get images
files = [os.path.join(args.folder, fn) for fn in os.listdir(args.folder)]
images = [fn for fn in files if os.path.splitext(fn)[1].lower() in ('.jpg', '.jpeg', '.png')]
if not images:
print('No images for making collage! Please select other directory with images!')
exit(1)
# shuffle images if needed
if args.shuffle:
random.shuffle(images)
print('Making collage...')
res = make_collage(images, args.output, args.width, args.init_height)
if not res:
print('Failed to create collage!')
exit(1)
print('Collage is ready!')
if __name__ == '__main__':
main()
The easiest way would be to use glob.glob() along with os.listdir(), although glob() uses bash syntax so you'll need to enter *_us*.
First import it:
from glob import glob
Then add a "pattern" optional positional arg:
parse.add_argument('-p', '--pattern', nargs="?", default=None, help="enter a grep-like expansion.")
Finally, change the line under # get images to something like this:
if args.pattern:
files = [fn for fn in glob(os.path.join(args.folder, args.pattern))]
else:
files = [os.path.join(args.folder, fn) for fn in os.listdir(args.folder)]
I'm trying to make speech recognition system with tensorflow.
Input data is an numpy array of size 50000 X 1.
Output data (mapping data) is an numpy array of size 400 X 1.
Input and mapping data is passed in batches of 2 in a list.
I've used this tutorial to design the neural network. Following is the code snippet:
For RNN:
input_data = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_DATA, sound_constants.MAX_COLUMN_SIZE_IN_DATA], name="train_input")
target = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT], name="train_output")
fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating one backward cell
bkwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating bidirectional RNN
val, _, _ = tf.nn.static_bidirectional_rnn(fwd_cell, bkwd_cell, tf.unstack(input_data), dtype=tf.float32)
For feeding data:
feed = {g['input_data'] : trb[0], g['target'] : trb[1], g['dropout'] : 0.6}
accuracy_, _ = sess.run([g['accuracy'], g['ts']], feed_dict=feed)
accuracy += accuracy_
When I ran the code, I got this error:
Traceback (most recent call last):
File "/home/wolborg/PycharmProjects/speech-to-text-rnn/src/rnn_train_1.py", line 205, in <module>
tr_losses, te_losses = train_network(g)
File "/home/wolborg/PycharmProjects/speech-to-text-rnn/src/rnn_train_1.py", line 177, in train_network
accuracy_, _ = sess.run([g['accuracy'], g['ts']], feed_dict=feed)
File "/home/wolborg/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 895, in run
run_metadata_ptr)
File "/home/wolborg/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1102, in _run
raise ValueError('Tensor %s may not be fed.' % subfeed_t)
ValueError: Tensor Tensor("Const:0", shape=(), dtype=float32) may not be fed.
Process finished with exit code 1
Earlier, I was facing this issue with tf.sparse_placeholder, then after some browsing, I changed input type to tf.placeholder and made related changes. Now I'm clueless on where I'm making the error.
Please suggest something as how should I feed data.
Entire code:
import tensorflow as tf
# for taking MFCC and label input
import numpy as np
import rnn_input_data_1
import sound_constants
# input constants
# Training Parameters
num_input = 10 # mfcc data input
training_data_size = 8 # determines number of files in training and testing module
testing_data_size = num_input - training_data_size
# Network Parameters
learning_rate = 0.0001 # for large training set, it can be set 0.001
num_hidden = 200 # number of hidden layers
num_classes = 28 # total alphabet classes (a-z) + extra symbols (', ' ')
epoch = 1 # number of iterations
batch_size = 2 # number of batches
mfcc_coeffs, text_data = rnn_input_data_1.mfcc_and_text_encoding()
class DataGenerator:
def __init__(self, data_size):
self.ptr = 0
self.epochs = 0
self.data_size = data_size
def next_batch(self):
self.ptr += batch_size
if self.ptr > self.data_size:
self.epochs += 1
self.ptr = 0
return mfcc_coeffs[self.ptr-batch_size : self.ptr], text_data[self.ptr-batch_size : self.ptr]
def reset_graph():
if 'sess' in globals() and sess:
sess.close()
tf.reset_default_graph()
def struct_network():
print ('Inside struct network !!')
reset_graph()
input_data = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_DATA, sound_constants.MAX_COLUMN_SIZE_IN_DATA], name="train_input")
target = tf.placeholder(tf.float32, [batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT], name="train_output")
keep_prob = tf.constant(1.0)
fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating one backward cell
bkwd_cell = tf.nn.rnn_cell.BasicLSTMCell(num_hidden, state_is_tuple=True, forget_bias=1.0)
# creating bidirectional RNN
val, _, _ = tf.nn.static_bidirectional_rnn(fwd_cell, bkwd_cell, tf.unstack(input_data), dtype=tf.float32)
# adding dropouts
val = tf.nn.dropout(val, keep_prob)
val = tf.transpose(val, [1, 0, 2])
last = tf.gather(val, int(val.get_shape()[0]) - 1)
# creating bidirectional RNN
print ('BiRNN created !!')
print ('Last Size: ', last.get_shape())
weight = tf.Variable(tf.truncated_normal([num_hidden * 2, sound_constants.MAX_ROW_SIZE_IN_TXT]))
bias = tf.Variable(tf.constant(0.1, shape=[sound_constants.MAX_ROW_SIZE_IN_TXT]))
# mapping to 28 output classes
logits = tf.matmul(last, weight) + bias
prediction = tf.nn.softmax(logits)
prediction = tf.reshape(prediction, shape = [batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT])
# getting probability distribution
mat1 = tf.cast(tf.argmax(prediction,1),tf.float32)
correct = tf.equal(prediction, target)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
logits = tf.reshape(logits, shape=[batch_size, sound_constants.MAX_ROW_SIZE_IN_TXT, sound_constants.MAX_COLUMN_SIZE_IN_TXT])
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=target))
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)
# returning components as dictionary elements
return {'input_data' : input_data,
'target' : target,
'dropout': keep_prob,
'loss': loss,
'ts': train_step,
'preds': prediction,
'accuracy': accuracy
}
def train_network(graph):
# initialize tensorflow session and all variables
# tf_gpu_config = tf.ConfigProto(allow_soft_placement = True, log_device_placement = True)
# tf_gpu_config.gpu_options.allow_growth = True
# with tf.Session(config = tf_gpu_config) as sess:
with tf.Session() as sess:
train_instance = DataGenerator(training_data_size)
test_instance = DataGenerator(testing_data_size)
print ('Training data size: ', train_instance.data_size)
print ('Testing data size: ', test_instance.data_size)
sess.run(tf.global_variables_initializer())
print ('Starting session...')
step, accuracy = 0, 0
tr_losses, te_losses = [], []
current_epoch = 0
while current_epoch < epoch:
step += 1
trb = train_instance.next_batch()
feed = {g['input_data'] : trb[0], g['target'] : trb[1], g['dropout'] : 0.6}
accuracy_, _ = sess.run([g['accuracy'], g['ts']], feed_dict=feed)
accuracy += accuracy_
if train_instance.epochs > current_epoch:
current_epoch += 1
tr_losses.append(accuracy / step)
step, accuracy = 0, 0
#eval test set
te_epoch = test_instance.epochs
while test_instance.epochs == te_epoch:
step += 1
print ('Testing round ', step)
trc = test_instance.next_batch()
feed = {g['input_data']: trc[0], g['target']: trc[1]}
accuracy_ = sess.run([g['accuracy']], feed_dict=feed)[0]
accuracy += accuracy_
te_losses.append(accuracy / step)
step, accuracy = 0,0
print("Accuracy after epoch", current_epoch, " - tr:", tr_losses[-1], "- te:", te_losses[-1])
return tr_losses, te_losses
g = struct_network()
tr_losses, te_losses = train_network(g)
You defined keep_prob as a tf.constant, but then trying to feed the value into it. Replace keep_prob = tf.constant(1.0) with keep_prob = tf.placeholder(tf.float32,[]) or keep_prob = tf.placeholder_with_default(1.0,[])
First of all I am pretty new at python, so bear with me. I am attempting to read from one file, retrieve specific values and overwrite old values in another file with a similar format. The format is 'text value=xxx' in both files. I have the first half of the program working, I can extract the values I want and have placed them into a dict named 'params{}'. The part I haven't figured out is how to just write the specific value into the target file without it showing up at the end of the file or just writing garbage or only half of the file. Here is my source code so far:
import os, os.path, re, fileinput, sys
#set the path to the resource files
#res_files_path = r'C:\Users\n518013\Documents\203-104 WA My MRT Files\CIA Data\pelzer_settings'
tst_res_files_path = r'C:\resource'
# Set path to target files.
#tar_files_path = r'C:\Users\n518013\Documents\203-104 WA My MRT Files\CIA Data\CIA3 Settings-G4'
tst_tar_files_path = r'C:\target'
#test dir.
test_files_path = r'C:\Users\n518013\Documents\MRT Equipment - BY 740-104 WA\CIA - AS\Setting Files\305_70R_22_5 setting files\CIA 1 Standard'
# function1 to find word index and point to value
def f_index(lst, item):
ind = lst.index(item)
val = lst[ind + 3]
print val
return val
# function 2 for values only 1 away from search term
def f_index_1(lst, item):
ind = lst.index(item)
val = lst[ind + 1]
return val
# Create file list.
file_lst = os.listdir(tst_res_files_path)
# Traverse the file list and read in dim settings files.
# Set up dict.
params = {}
#print params
for fname in file_lst:
file_loc = os.path.join(tst_res_files_path, fname)
with open(file_loc, 'r') as f:
if re.search('ms\.', fname):
print fname
break
line = f.read()
word = re.split('\W+', line)
print word
for w in word:
if w == 'Number':
print w
params['sectors'] = f_index(word, w)
elif w == 'Lid':
params['lid_dly'] = f_index(word, w)
elif w == 'Head':
params['rotation_dly'] = f_index(word, w)
elif w == 'Horizontal':
tmp = f_index_1(word, w)
param = int(tmp) + 72
params['horizontal'] = str(param)
elif w == 'Vertical':
tmp = f_index_1(word, w)
param = int(tmp) - 65
params['vertical'] = str(param)
elif w == 'Tilt':
params['tilt'] = f_index_1(word, w)
else:
print 'next...'
print params #this is just for debugging
file_tar = os.path.join(tst_tar_files_path, fname)
for lines in fileinput.input(file_tar, inplace=True):
print lines.rstrip()
if lines.startswith('Number'):
if lines[-2:-1] != params['sectors']:
repl = params['sectors']
lines = lines.replace(lines[-2:-1], repl)
sys.stdout.write(lines)
else:
continue
Sample text files:
[ADMINISTRATIVE SETTINGS]
SettingsType=SingleScan
DimensionCode=
Operator=
Description=rev.1 4sept03
TireDimClass=Crown
TireWidth=400mm
[TEST PARAMETERS]
Number Of Sectors=9
Vacuum=50
[DELAY SETTINGS]
Lid Close Delay=3
Head Rotation Delay=3
[HEAD POSITION]
Horizontal=140
Vertical=460
Tilt=0
[CALIBRATION]
UseConvFactors=0
LengthUnit=0
ConvMMX=1
ConvPixelX=1
CorrFactorX=1
ConvMMY=1
ConvPixelY=1
CorrFactorY=1
end sample txt.
The code I have only writes about half of the file back, and I don't understand why? I am trying to replace the line 'Number of Sectors=9' with 'Number of Sectors=8' if I could get this to work, the rest of the replacements can be done using if statements.
Please help! I've spent hours on google looking for answers and info and everything I find gets me close but no cigar!
Thank you all in advance!
your file has the '.ini' format. python supports reading and writing those with the ConfigParser module. you could do this:
# py3: from pathlib import Path
import os.path
import configparser
# py3: IN_PATH = Path(__file__).parent / '../data/sample.ini'
# py3: OUT_PATH = Path(__file__).parent / '../data/sample_out.ini'
HERE = os.path.dirname(__file__)
IN_PATH = os.path.join(HERE, '../data/sample.ini')
OUT_PATH = os.path.join(HERE, '../data/sample_out.ini')
config = configparser.ConfigParser()
# py3: config.read(str(IN_PATH))
config.read(IN_PATH)
print(config['CALIBRATION']['LengthUnit'])
config['CALIBRATION']['LengthUnit'] = '27'
# py3: with OUT_PATH.open('w') as fle:
with open(OUT_PATH, 'w') as fle:
config.write(fle)