Pytorch error when training an image captioning model Dimension out of range - computer-vision

Question

i was training a model, but it say that it is not enough values to unpack at the decoder. Here is the training step
def train(loader, encoder, decoder, optimizer, criterion, epoch):
loss_tracker, acc_tracker = AvgMeter(), AvgMeter()
encoder.train()
decoder.train()
pbar = tqdm.tqdm(enumerate(loader), total=len(loader))
for i, (images, target_sequences, sequence_lengths) in pbar:
images = images.to(device)
target_sequences = target_sequences.to(device)
sequence_lengths = sequence_lengths.to(device)
# Forward prop.
logits, alphas, sorted_target_sequences, sorted_decode_lengths, sorted_indices = decoder(images, target_sequences, sequence_lengths)
# Since we decoded starting with <sos>, the targets are all words after <sos>, up to <eos>
sorted_target_sequences = sorted_target_sequences[:, 1:]
# Remove paddings
logits = pack_padded_sequence(logits, sorted_decode_lengths).data
sorted_target_sequences = pack_padded_sequence(sorted_target_sequences, sorted_decode_lengths, batch_first=True).data
# Calculate loss
loss = criterion(logits, sorted_target_sequences)
# Add doubly stochastic attention regularization
loss += alpha_c * ((1. - alphas.sum(dim=1)) ** 2).mean()
# Back prop.
optimizer.zero_grad()
loss.backward()
# Clip gradients
if grad_clip is not None:
clip_gradient(optimizer, grad_clip)
# Update weights
optimizer.step()
# Track metrics
loss_tracker.update(loss.item(), sum(sorted_decode_lengths))
acc_tracker.update(accuracy(logits, sorted_target_sequences, 5), sum(sorted_decode_lengths))
# Update progressbar description
pbar.set_description(f'Epoch: {epoch + 1:03d} - train_loss: {loss_tracker.avg:.3f} - train_acc: {acc_tracker.avg:.3f}%')
return loss_tracker.avg, acc_tracker.avg
# , train_bleu1, train_bleu2, train_bleu3, train_bleu4
I was trying to solve the error by change the images, target_seq, and length into for i, (images, (target_sequences, sequence_lengths)) in pbar: then the error continue to it
<ipython-input-89-edf18c02b224> in process_training(encoder, decoder, optimizer, criterion, train_loader, valid_loader, field, alpha_c, start_epoch, n_epochs, grad_clip, model_name, last_improv)
28 # Train step
29 train_loss, train_acc = train(loader=train_loader, encoder=encoder, decoder=decoder, optimizer=optimizer, criterion=criterion,
---> 30 epoch=epoch)
31 # Validation step
32 val_loss, val_acc, bleu1, bleu2, bleu3, bleu4 = validate(encoder=encoder,decoder=decoder, criterion=criterion, loader=valid_loader,
<ipython-input-102-96da9ec6e1d4> in train(loader, encoder, decoder, optimizer, criterion, epoch)
9 sequence_lengths = sequence_lengths.to(device)
10 # Forward prop.
---> 11 logits, alphas, sorted_target_sequences, sorted_decode_lengths, sorted_indices = decoder(images, target_sequences, sequence_lengths)
12 # Since we decoded starting with <sos>, the targets are all words after <sos>, up to <eos>
13 sorted_target_sequences = sorted_target_sequences[:, 1:]
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1129 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130 return forward_call(*input, **kwargs)
1131 # Do not call functions when jit is used
1132 full_backward_hooks, non_full_backward_hooks = [], []
<ipython-input-97-da50cbd97d62> in forward(self, encoder_out, encoded_captions, caption_lengths)
88 # caption_lengths, sort_ind = caption_lengths.squeeze(1).sort(dim=0, descending=True)
89
---> 90 caption_lengths, sort_ind = caption_lengths.squeeze(1). sort(dim=0, descending=True)
91
92
IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)
Here is the code i use in the decoder to forward the input. It's error in the line of caption_lengths, sort_ind = caption_lengths.squeeze(1).sort(dim=0, descending=True) to sort caption data
def forward(self, encoder_out, encoded_captions, caption_lengths):
"""
Forward propagation.
:param encoder_out: encoded images, a tensor of dimension (batch_size, enc_image_size, enc_image_size, encoder_dim)
:param encoded_captions: encoded captions, a tensor of dimension (batch_size, max_caption_length)
:param caption_lengths: caption lengths, a tensor of dimension (batch_size, 1)
:return: scores for vocabulary, sorted encoded captions, decode lengths, weights, sort indices
"""
batch_size = encoder_out.size(0)
encoder_dim = encoder_out.size(-1)
vocab_size = self.vocab_size
# Flatten image
encoder_out = encoder_out.view(batch_size, -1, encoder_dim) # (batch_size, num_pixels, encoder_dim)
num_pixels = encoder_out.size(1)
# Sort input data by decreasing lengths; why? apparent below
caption_lengths, sort_ind = caption_lengths.squeeze(1).sort(dim=0, descending=True)
encoder_out = encoder_out[sort_ind]
encoded_captions = encoded_captions[sort_ind]
# Embedding
embeddings = self.embedding(encoded_captions) # (batch_size, max_caption_length, embed_dim)
# Initialize LSTM state
h, c = self.init_hidden_state(encoder_out) # (batch_size, decoder_dim)
# We won't decode at the <end> position, since we've finished generating as soon as we generate <end>
# So, decoding lengths are actual lengths - 1
decode_lengths = (caption_lengths - 1).tolist()
# Create tensors to hold word predicion scores and alphas
predictions = torch.zeros(batch_size, max(decode_lengths), vocab_size).to(device)
alphas = torch.zeros(batch_size, max(decode_lengths), num_pixels).to(device)
# At each time-step, decode by
# attention-weighing the encoder's output based on the decoder's previous hidden state output
# then generate a new word in the decoder with the previous word and the attention weighted encoding
for t in range(max(decode_lengths)):
batch_size_t = sum([l > t for l in decode_lengths])
attention_weighted_encoding, alpha = self.attention(encoder_out[:batch_size_t],
h[:batch_size_t])
gate = self.sigmoid(self.f_beta(h[:batch_size_t])) # gating scalar, (batch_size_t, encoder_dim)
attention_weighted_encoding = gate * attention_weighted_encoding
h, c = self.decode_step(
torch.cat([embeddings[:batch_size_t, t, :], attention_weighted_encoding], dim=1),
(h[:batch_size_t], c[:batch_size_t])) # (batch_size_t, decoder_dim)
preds = self.fc(self.dropout(h)) # (batch_size_t, vocab_size)
predictions[:batch_size_t, t, :] = preds
alphas[:batch_size_t, t, :] = alpha
return predictions, encoded_captions, decode_lengths, alphas,