std::vector.push_back() throwing badalloc exception - c++

Question

I started writing a lexer, and I made the following file to test that everything so far is working okay:
Main.cpp
#include <iostream>
#include "Lexer.h"
#include "Token.h"
int main(void)
{
std::string str(""); // I'll use this to test expressions
Lexer lexer(str);
std::vector<Token> tokens = lexer.lex();
for(auto it = tokens.begin(); it != tokens.end(); ++it)
{
std::string str;
switch(it->type)
{
case TokenType::_EOF:
str = "EOF";
break;
case TokenType::ERROR:
str = "ERROR";
break;
case TokenType::SEMICOLON:
str = "SEMICOLON";
break;
case TokenType::PLUS:
str = "PLUS";
break;
case TokenType::LESS_THAN:
str = "LESS_THAN";
break;
case TokenType::GREATER_THAN:
str = "GREATER_THAN";
break;
case TokenType::INT:
str = "INT";
break;
case TokenType::ID:
str = "ID";
break;
case TokenType::WHITESPACE:
str = "WHITESPACE";
break;
default:
str = "<Unknown Token>";
}
std::cout << str << ", detail='" << it->detail << "'" << std::endl;
}
return 0;
}
The line lexer.lex() throws an exception. Looking in Lexer.h:
std::vector<Token> Lexer::lex(void)
{
// Reset input pointer
inputPtr = 0;
updateCurrentChar();
// Read tokens until EOF
std::vector<Token> tokens;
Token *token = nullptr;
do
{
token = getNext();
tokens.push_back(*token);
} while(token->type != TokenType::_EOF);
return tokens;
}
The line tokens.push_back(*token) is throwing an exception:
I tried looking at the info on push_back() here, and saw that:
If a reallocation happens, the storage is allocated using the container's allocator, which may throw exceptions on failure (for the default allocator, bad_alloc is thrown if the allocation request does not succeed).
This seems to be my problem, but I don't understand why the allocation request wouldn't succeed.
For completeness, here are all the files:
Token.h
#pragma once
#include <string>
enum class TokenType
{
_EOF,
ERROR,
EQUALS,
SEMICOLON,
PLUS,
LESS_THAN,
GREATER_THAN,
INT,
ID,
WHITESPACE
};
struct Token
{
TokenType type;
std::string detail;
};
Lexer.h
#pragma once
#include <string>
#include <vector>
#include "Token.h"
class Lexer
{
public:
Lexer(std::string);
~Lexer(void);
std::vector<Token> lex(void);
private:
Token* getNext(void);
void updateCurrentChar(void);
void increment(void);
bool matchCurrent(char);
bool isWhitespace(char) const;
bool isDigit(char) const;
bool isLetter(char) const;
bool isLowercaseLetter(char) const;
bool isUppercaseLetter(char) const;
std::string readWhitespace(void);
std::string readInt(void);
std::string readId(void);
std::string input;
int inputPtr;
char currentChar;
const char EOF_CHAR;
};
Lexer.cpp
#include "Lexer.h"
Lexer::Lexer(std::string _input)
: input(_input), EOF_CHAR(-1)
{
}
Lexer::~Lexer(void)
{
}
std::vector<Token> Lexer::lex(void)
{
// Reset input pointer
inputPtr = 0;
updateCurrentChar();
// Read tokens until EOF
std::vector<Token> tokens;
Token *token = nullptr;
do
{
token = getNext();
tokens.push_back(*token);
} while(token->type != TokenType::_EOF);
return tokens;
}
void Lexer::updateCurrentChar(void)
{
currentChar = inputPtr < input.length()
? input[inputPtr]
: EOF_CHAR;
}
void Lexer::increment(void)
{
inputPtr++;
updateCurrentChar();
}
bool Lexer::matchCurrent(char toMatch)
{
if(toMatch == currentChar)
{
increment();
return true;
}
return false;
}
Token* Lexer::getNext(void)
{
Token token;
if(isWhitespace(currentChar))
{
token.type = TokenType::WHITESPACE;
token.detail = readWhitespace();
return &token;
}
if(isDigit(currentChar))
{
token.type = TokenType::INT;
token.detail = readInt();
return &token;
}
if(isLetter(currentChar))
{
token.type = TokenType::ID;
token.detail = readId();
return &token;
}
if(currentChar == EOF_CHAR)
{
token.type = TokenType::_EOF;
return &token;
}
switch(currentChar)
{
case ';':
token.type = TokenType::SEMICOLON;
case '=':
token.type = TokenType::EQUALS;
case '<':
token.type = TokenType::LESS_THAN;
case '>':
token.type = TokenType::GREATER_THAN;
case '+':
token.type = TokenType::PLUS;
default:
token.type = TokenType::ERROR;
token.detail = currentChar;
}
increment();
return &token;
}
std::string Lexer::readWhitespace(void)
{
std::string ws;
while(isWhitespace(currentChar))
{
ws += currentChar;
increment();
}
return ws;
}
std::string Lexer::readInt(void)
{
std::string ws;
while(isDigit(currentChar))
{
ws += currentChar;
increment();
}
return ws;
}
std::string Lexer::readId(void)
{
std::string ws;
while(isWhitespace(currentChar))
{
ws += currentChar;
increment();
}
return ws;
}
bool Lexer::isDigit(char c) const
{
return c >= '0' && c <= '9';
}
bool Lexer::isLetter(char c) const
{
return isLowercaseLetter(c)
|| isUppercaseLetter(c);
}
bool Lexer::isLowercaseLetter(char c) const
{
return c >= 'a' && c <= 'z';
}
bool Lexer::isUppercaseLetter(char c) const
{
return c >= 'A' && c <= 'Z';
}
bool Lexer::isWhitespace(char c) const
{
switch(c)
{
case ' ':
case '\n':
case '\t':
case '\r':
return true;
default:
return false;
}
}

Answer 1

The problem is that you in getNext you return a pointer to a local variable. Remember that when the function returns all local variables are destructed and so the pointer now points to a destructed object. Or, since local variables are on the stack and the stack is reused between function calls, it can now point to something completely different. The result of you then dereferencing this now invalid pointer leads to undefined behavior, and it's common for undefined behavior to cause crashes, but it may also seem to work but the data is completely screwed up.
The obvious solution is of course to return a copy of the object, i.e. don't use pointers (which is a good tip in general).