I am writing a multi-threaded XML parser using Xercesc and I have followed the guide in https://xerces.apache.org/xerces-c/faq-parse-3.html#faq-6 by declaring the XMLPlatformUtils::Initialize() and XMLPlatformUtils::Terminate(); in the main thread while creating new parser and handler that can parse different XML files concurrently. The threads are created this way
std::vector<std::thread> thread_vector;
for (std::string i: input_folder){
SAXParser* parser = new SAXParser();
parser->setValidationScheme(SAXParser::ValSchemes::Val_Never);
parser->setDoSchema(false);
parser->setValidationSchemaFullChecking(false);
parser->cacheGrammarFromParse(false);
parser->setDoNamespaces(true); // optional
SAXHandler* handler= new SAXHandler();
parser->setDocumentHandler(handler);
parser->setErrorHandler(handler);
thread_vector.emplace_back(std::thread(parseFile,i,outputDir,handler,parser));
// parseFile(i,outputDir,handler,parser);
}
std::for_each(thread_vector.begin(),thread_vector.end(),std::mem_fn(&std::thread::join));
Inside the handle function I output some of the parsed items to an output file using std::ofstream and also use XMLString::transcode() to transcode XML strings. Although the result of the code running in single thread and multi-thread is similar, I find errors when I'm running it on valgrind with --tool=drd. Here is some of the error
==1045== Thread 7:
==1045== Conflicting load by thread 7 at 0x00126098 size 8
==1045== at 0x4D5E6E3: std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x111DA8: parseFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x112A77: std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*> > >::_M_run() (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x4D01DE3: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x48424BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== by 0x4E30608: start_thread (pthread_create.c:477)
==1045== by 0x4F6A132: clone (clone.S:95)
==1045== Allocation context: BSS section of /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser
==1045== Other segment start (thread 2)
==1045== at 0x4F6A125: clone (clone.S:78)
==1045== Other segment end (thread 2)
==1045== at 0x484687C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== by 0x49D48AF: xercesc_3_2::XMLMutexLock::~XMLMutexLock() (Mutexes.cpp:84)
==1045== by 0x4B26C76: xercesc_3_2::ICULCPTranscoder::transcode(char16_t const*, xercesc_3_2::MemoryManager*) (ICUTransService.cpp:949)
==1045== by 0x4B2810E: xercesc_3_2::PosixFileMgr::fileOpen(char16_t const*, bool, xercesc_3_2::MemoryManager*) (PosixFileMgr.cpp:59)
==1045== by 0x49D2D8F: xercesc_3_2::BinFileInputStream::BinFileInputStream(char16_t const*, xercesc_3_2::MemoryManager*) (BinFileInputStream.cpp:40)
==1045== by 0x4A2C2E2: xercesc_3_2::LocalFileInputSource::makeStream() const (LocalFileInputSource.cpp:162)
==1045== by 0x4A5C237: xercesc_3_2::ReaderMgr::createReader(xercesc_3_2::InputSource const&, bool, xercesc_3_2::XMLReader::RefFrom, xercesc_3_2::XMLReader::Types, xercesc_3_2::XMLReader::Sources, bool, unsigned long) (ReaderMgr.cpp:379)
==1045== by 0x4A586CE: xercesc_3_2::IGXMLScanner::scanReset(xercesc_3_2::InputSource const&) (IGXMLScanner2.cpp:1287)
==1045== by 0x4A79C0D: xercesc_3_2::XMLScanner::scanFirst(xercesc_3_2::InputSource const&, xercesc_3_2::XMLPScanToken&) (XMLScanner.cpp:538)
==1045== by 0x4A7AE49: xercesc_3_2::XMLScanner::scanFirst(char16_t const*, xercesc_3_2::XMLPScanToken&) (XMLScanner.cpp:514)
==1045== by 0x4A7B1C9: xercesc_3_2::XMLScanner::scanFirst(char const*, xercesc_3_2::XMLPScanToken&) (XMLScanner.cpp:523)
==1045== by 0x111B65: parseFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== Other segment start (thread 3)
==1045== at 0x4F6A125: clone (clone.S:78)
==1045== Other segment end (thread 3)
==1045== at 0x484687C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== by 0x49D48AF: xercesc_3_2::XMLMutexLock::~XMLMutexLock() (Mutexes.cpp:84)
==1045== by 0x4B26C76: xercesc_3_2::ICULCPTranscoder::transcode(char16_t const*, xercesc_3_2::MemoryManager*) (ICUTransService.cpp:949)
==1045== by 0x11439B: SAXHandler::transcode[abi:cxx11](char16_t const*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x11F38D: SAXHandler::endElement(char16_t const*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x4AA4BC4: xercesc_3_2::SAXParser::endElement(xercesc_3_2::XMLElementDecl const&, unsigned int, bool, char16_t const*) (SAXParser.cpp:865)
==1045== by 0x4A4C588: xercesc_3_2::IGXMLScanner::scanEndTag(bool&) (IGXMLScanner.cpp:1178)
==1045== by 0x4A509FB: xercesc_3_2::IGXMLScanner::scanNext(xercesc_3_2::XMLPScanToken&) (IGXMLScanner.cpp:377)
==1045== by 0x111B86: parseFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x112A77: std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*> > >::_M_run() (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x4D01DE3: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x48424BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== Other segment start (thread 4)
==1045== at 0x4F6A125: clone (clone.S:78)
==1045== Other segment end (thread 4)
==1045== at 0x484687C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== by 0x49D48AF: xercesc_3_2::XMLMutexLock::~XMLMutexLock() (Mutexes.cpp:84)
==1045== by 0x4B26C76: xercesc_3_2::ICULCPTranscoder::transcode(char16_t const*, xercesc_3_2::MemoryManager*) (ICUTransService.cpp:949)
==1045== by 0x11439B: SAXHandler::transcode[abi:cxx11](char16_t const*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x1173D0: SAXHandler::startElement(char16_t const*, xercesc_3_2::AttributeList&) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x4AA48C8: xercesc_3_2::SAXParser::startElement(xercesc_3_2::XMLElementDecl const&, unsigned int, char16_t const*, xercesc_3_2::RefVectorOf<xercesc_3_2::XMLAttr> const&, unsigned long, bool, bool) (SAXParser.cpp:982)
==1045== by 0x4A4EED7: xercesc_3_2::IGXMLScanner::scanStartTagNS(bool&) (IGXMLScanner.cpp:2632)
==1045== by 0x4A509DA: xercesc_3_2::IGXMLScanner::scanNext(xercesc_3_2::XMLPScanToken&) (IGXMLScanner.cpp:386)
==1045== by 0x111B86: parseFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x112A77: std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*> > >::_M_run() (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x4D01DE3: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x48424BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== Other segment start (thread 5)
==1045== at 0x4F6A125: clone (clone.S:78)
==1045== Other segment end (thread 5)
==1045== at 0x484687C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== by 0x49D48AF: xercesc_3_2::XMLMutexLock::~XMLMutexLock() (Mutexes.cpp:84)
==1045== by 0x4B26C76: xercesc_3_2::ICULCPTranscoder::transcode(char16_t const*, xercesc_3_2::MemoryManager*) (ICUTransService.cpp:949)
==1045== by 0x4B2810E: xercesc_3_2::PosixFileMgr::fileOpen(char16_t const*, bool, xercesc_3_2::MemoryManager*) (PosixFileMgr.cpp:59)
==1045== by 0x49D2D8F: xercesc_3_2::BinFileInputStream::BinFileInputStream(char16_t const*, xercesc_3_2::MemoryManager*) (BinFileInputStream.cpp:40)
==1045== by 0x4A2C2E2: xercesc_3_2::LocalFileInputSource::makeStream() const (LocalFileInputSource.cpp:162)
==1045== by 0x4A5C237: xercesc_3_2::ReaderMgr::createReader(xercesc_3_2::InputSource const&, bool, xercesc_3_2::XMLReader::RefFrom, xercesc_3_2::XMLReader::Types, xercesc_3_2::XMLReader::Sources, bool, unsigned long) (ReaderMgr.cpp:379)
==1045== by 0x4A586CE: xercesc_3_2::IGXMLScanner::scanReset(xercesc_3_2::InputSource const&) (IGXMLScanner2.cpp:1287)
==1045== by 0x4A79C0D: xercesc_3_2::XMLScanner::scanFirst(xercesc_3_2::InputSource const&, xercesc_3_2::XMLPScanToken&) (XMLScanner.cpp:538)
==1045== by 0x4A7AE49: xercesc_3_2::XMLScanner::scanFirst(char16_t const*, xercesc_3_2::XMLPScanToken&) (XMLScanner.cpp:514)
==1045== by 0x4A7B1C9: xercesc_3_2::XMLScanner::scanFirst(char const*, xercesc_3_2::XMLPScanToken&) (XMLScanner.cpp:523)
==1045== by 0x111B65: parseFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== Other segment start (thread 6)
==1045== at 0x4F6A125: clone (clone.S:78)
==1045== Other segment end (thread 6)
==1045== at 0x4F590AF: __libc_write (write.c:26)
==1045== by 0x4F590AF: write (write.c:24)
==1045== by 0x4ED9E8C: _IO_file_write##GLIBC_2.2.5 (fileops.c:1181)
==1045== by 0x4EDB950: new_do_write (fileops.c:449)
==1045== by 0x4EDB950: _IO_new_do_write (fileops.c:426)
==1045== by 0x4EDB950: _IO_do_write##GLIBC_2.2.5 (fileops.c:423)
==1045== by 0x4EDBE92: _IO_file_overflow##GLIBC_2.2.5 (fileops.c:784)
==1045== by 0x4ED6972: putc (putc.c:31)
==1045== by 0x4D5E258: std::ostream::put(char) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x111B0D: parseFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x112A77: std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*> > >::_M_run() (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x4D01DE3: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x48424BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== by 0x4E30608: start_thread (pthread_create.c:477)
==1045== by 0x4F6A132: clone (clone.S:95)
==1045== Other segment start (thread 6)
==1045== at 0x4F6A125: clone (clone.S:78)
==1045== Other segment end (thread 6)
==1045== at 0x4F590AF: __libc_write (write.c:26)
==1045== by 0x4F590AF: write (write.c:24)
==1045== by 0x4ED9E8C: _IO_file_write##GLIBC_2.2.5 (fileops.c:1181)
==1045== by 0x4EDB950: new_do_write (fileops.c:449)
==1045== by 0x4EDB950: _IO_new_do_write (fileops.c:426)
==1045== by 0x4EDB950: _IO_do_write##GLIBC_2.2.5 (fileops.c:423)
==1045== by 0x4EDBE92: _IO_file_overflow##GLIBC_2.2.5 (fileops.c:784)
==1045== by 0x4ED6972: putc (putc.c:31)
==1045== by 0x4D5E258: std::ostream::put(char) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x111B0D: parseFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*) (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x112A77: std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, SAXHandler*, xercesc_3_2::SAXParser*> > >::_M_run() (in /mnt/c/Users/dennnn/Documents/Projects/ParserCPP/parser)
==1045== by 0x4D01DE3: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28)
==1045== by 0x48424BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
==1045== by 0x4E30608: start_thread (pthread_create.c:477)
==1045== by 0x4F6A132: clone (clone.S:95)
I wonder why an object that is declared inside a threaded object can still be accessed by threads from another object (which I thought should be independent from each other and in this case is the XMLString::transcode() and std::ofstream) and cause data race as well as how to approach this problem. Thank you.
UPDATE 24/06/2022
The ICULCPTranscoder::transcode or the object used to transcode the XMLCh* is declared in the XMLPlatformUtils::Initialize() and is shared among the threads so it will use mutex lock to operate. Also, it might be that the conflicting objects belong to the same cache line.
Related
I'm trying to do this:
int main(){
std::unique_ptr<std::thread> up(nullptr);
up = std::make_unique<std::thread>(upload, url, path);
return 0;
}
where, upload is basically a static non-member function and url and path are std::string. upload is defined as:
static void upload(const string url, const string path){ int sync_status;}
But this is always getting reported as a memory leak under Valgrind:
249 (80 direct, 169 indirect) bytes in 1 blocks are definitely lost in loss record 151 of 169
==12088== at 0x4C3017F: operator new(unsigned long) (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==12088== by 0x1395F9: _S_make_state<std::thread::_Invoker<std::tuple<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> , std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > (thread:197)
==12088== by 0x1395F9: thread<void (&)(std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char>), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> &> (thread:126)
==12088== by 0x1395F9: std::_MakeUniq<std::thread>::__single_object std::make_unique<std::thread, void (&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> ), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(void (&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> &, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) (unique_ptr.h:825)
==12088== by 0x131D8D: main
Your code ends up invoking std::thread::~thread which calls std::terminate because the thread is neither detached nor joined. This is why valgrind reports an error.
Either join or detach the thread.
I've recently updated my Gentoo Linux PC with GCC 5.4.0.
I also recompiled all packages depending on gcc (it was mentioned in gcc upgrade guide for gentoo).
Most of programs works well, however my tests, using cppunit, throw std::bad_alloc before starting.
At first I thought there might be a problem with updated version of cppunit, so I downgraded it to previous version, but the problem still exists.
Whenever I start any test, the application immediately throws:
terminate called after throwing an instance of 'std::bad_alloc'
what(): std::bad_alloc
I started the application in gdb and printed debug backtrace:
#0 0x00007ffff545d218 in raise () from /lib64/libc.so.6
#1 0x00007ffff545e69a in abort () from /lib64/libc.so.6
#2 0x00007ffff5d8f18d in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/libstdc++.so.6
#3 0x00007ffff5d8d046 in ?? () from /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/libstdc++.so.6
#4 0x00007ffff5d8d091 in std::terminate() () from /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/libstdc++.so.6
#5 0x00007ffff5d8d297 in __cxa_throw () from /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/libstdc++.so.6
#6 0x00007ffff5db5f52 in std::__throw_bad_alloc() () from /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/libstdc++.so.6
#7 0x0000000000b2b3ae in __gnu_cxx::new_allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >::allocate
(this=0x7fffffffda58, __n=18446741874689290911) at /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/include/g++-v5/ext/new_allocator.h:102
#8 0x0000000000b2b1e2 in std::allocator_traits<std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::allocate (__a=..., __n=18446741874689290911) at /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/include/g++-v5/bits/alloc_traits.h:491
#9 0x0000000000b2b09e in std::__cxx1998::_Vector_base<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_allocate (this=0x7fffffffda58,
__n=18446741874689290911) at /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/include/g++-v5/bits/stl_vector.h:170
#10 0x0000000000b2af71 in std::__cxx1998::_Vector_base<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_create_storage (this=0x7fffffffda58,
__n=18446741874689290911) at /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/include/g++-v5/bits/stl_vector.h:185
#11 0x0000000000b2acf1 in std::__cxx1998::_Vector_base<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Vector_base (this=0x7fffffffda58,
__n=18446741874689290911, __a=...) at /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/include/g++-v5/bits/stl_vector.h:136
#12 0x0000000000b29c9c in std::__cxx1998::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::vector (this=0x7fffffffda58,
__x=std::__cxx1998::vector of length -2199020260704, capacity -2199020259433 = {...}) at /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/include/g++-v5/bits/stl_vector.h:320
#13 0x0000000000b29449 in std::__debug::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::vector (this=0x7fffffffda40)
at /usr/lib/gcc/x86_64-pc-linux-gnu/5.4.0/include/g++-v5/debug/vector:193
#14 0x0000000000b294ef in CppUnit::TestSuiteBuilderContextBase::TestSuiteBuilderContextBase (this=0x7fffffffda20) at /usr/include/cppunit/extensions/TestSuiteBuilderContext.h:29
#15 0x0000000000b29515 in CppUnit::TestSuiteBuilderContext<Types_Decimal>::TestSuiteBuilderContext (this=0x7fffffffda20, contextBase=...) at /usr/include/cppunit/extensions/TestSuiteBuilderContext.h:106
#16 0x0000000000b28a16 in Types_Decimal::addTestsToSuite (baseContext=...) at ./Tests/../../pCpp/CppUnitTests/Types_Decimal.hpp:10
#17 0x0000000000b28caa in Types_Decimal::suite () at ./Tests/../../pCpp/CppUnitTests/Types_Decimal.hpp:12
#18 0x0000000000b2871f in main () at Types_Decimal.cpp:8
There is no problem with test file itself, because I made no changes to it, and lots of test were working for long time.
After looking into stack trace, I'm pretty curious what happened on frame #12: __x=std::**__cxx1998**::vector of length **-2199020260704**, capacity **-2199020259433** = {...}
Why there is cxx1998? Both length and capacity looks pretty insane (uninitialized?).
TestSuiteBuilderContext.h:29:
void
TestSuiteBuilderContextBase::addTest( Test *test )
{
m_suite.addTest( test ); //line 29
}
TestSuite.cpp:
/// Adds a test to the suite.
void
TestSuite::addTest( Test *test )
{
m_tests.push_back( test );
}
I came up with partial solution.
It looks like the compile option:
-D_GLIBCXX_DEBUG
Was causing the problem. After removing it - tests run succesfully.
I am trying to control de workflow of a program I am developing. To do so I have a map< unsigned int, list < unsigned int > > in which the first key will be the id and the second ( the list ) will be used to know if I end correctly all tasks. The only operations I use on this list are:
myMap[iD].size()
myMap[iD].push_back(foo) <- ( foo is an unsigned int )
for (std::list<unsigned int>::iterator it=myMap[iD].begin(); it != myMap[iD].end(); ++it){
myMap[iD].erase(it)
}
The length of my map can grow to 1452 elements and each element list size can be from the order of 1000 ~ 5000.
When I run the program sometimes I receive a segmentation fault and some times a bad allocation error. My guess is that this come from the push_back because:
If I don't push back any element the program works fine.
The storage for the new elements is allocated using the container's allocator, which may throw exceptions on failure (for the default allocator, bad_alloc is thrown if the allocation request does not succeed). http://www.cplusplus.com/reference/list/list/push_back/
This is the only part of the code where I use the map:
if (FOO != 0){
if (PID != 0){
if ( myMap.size() + 5 < myMap.max_size()){
if (myMap[PID].size() > 1000) myMap[PID].pop_front();
myMap[PID].push_back(EVENTVALUE);
}
}
} else {
if (PID != 0 and foo2 != 0 and myMap.find(PID) != myMap.end()) {
for (std::list<unsigned int>::iterator it=myMap[PID].begin(); it != myMap[PID].end(); ++it){
if (*it == foo2){
cout << " erasing pid: " << PID << endl;
myMap[PID].erase(it);
if ( myMap[PID].size() == 0 ) myMap.erase(PID);
break;
}
}
}
}
I've also tried to use the tool Valgrind and this is the output:
==4092== Invalid read of size 8
==4092== at 0x4F09EB8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::string const&) (in /usr/lib64/libstdc++.so.6.0.21)
==4092== by 0x40CCA9: construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (new_allocator.h:120)
==4092== by 0x40CCA9: _S_construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (alloc_traits.h:254)
==4092== by 0x40CCA9: construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (alloc_traits.h:393)
==4092== by 0x40CCA9: std::vector<std::string, std::allocator<std::string> >::push_back(std::string const&) (stl_vector.h:905)
==4092== by 0x4157AC: foo::foo(std::basic_ofstream<char, std::char_traits<char> >&) (foo.cc:1743)
==4092== by 0x404F49: main (foo.cc:3159)
==4092== Address 0x6157d08 is 0 bytes after a block of size 8 alloc'd
==4092== at 0x4C29670: operator new(unsigned long) (in /usr/lib64/valgrind/vgpreload_memcheck-amd64-linux.so)
==4092== by 0x40DB77: allocate (new_allocator.h:104)
==4092== by 0x40DB77: _M_allocate (stl_vector.h:168)
==4092== by 0x40DB77: void std::vector<std::string, std::allocator<std::string> >::_M_emplace_back_aux<std::string>(std::string&&) (vector.tcc:404)
==4092== by 0x408F3E: push_back (stl_vector.h:920)
==4092== by 0x408F3E: split(std::string const&, char, int) (foo.cc:416)
==4092== by 0x41577F: lustreLine::toPRV(std::basic_ofstream<char, std::char_traits<char> >&) (foo.cc:1741)
==4092== by 0x404F49: main (foo.cc:3159)
==4092==
==4092== Invalid read of size 4
==4092== at 0x4F09EBB: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::string const&) (in /usr/lib64/libstdc++.so.6.0.21)
==4092== by 0x40CCA9: construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (new_allocator.h:120)
==4092== by 0x40CCA9: _S_construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (alloc_traits.h:254)
==4092== by 0x40CCA9: construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (alloc_traits.h:393)
==4092== by 0x40CCA9: std::vector<std::string, std::allocator<std::string> >::push_back(std::string const&) (stl_vector.h:905)
==4092== by 0x4157AC: foo::foo(std::basic_ofstream<char, std::char_traits<char> >&) (foo.cc:1743)
==4092== by 0x404F49: main (foo.cc:3159)
==4092== Address 0xfffffffffffffff8 is not stack'd, malloc'd or (recently) free'd
==4092==
==4092==
==4092== Process terminating with default action of signal 11 (SIGSEGV)
==4092== Access not within mapped region at address 0xFFFFFFFFFFFFFFF8
==4092== at 0x4F09EBB: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::string const&) (in /usr/lib64/libstdc++.so.6.0.21)
==4092== by 0x40CCA9: construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (new_allocator.h:120)
==4092== by 0x40CCA9: _S_construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (alloc_traits.h:254)
==4092== by 0x40CCA9: construct<std::basic_string<char>, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&> (alloc_traits.h:393)
==4092== by 0x40CCA9: std::vector<std::string, std::allocator<std::string> >::push_back(std::string const&) (stl_vector.h:905)
==4092== by 0x4157AC: foo::foo(std::basic_ofstream<char, std::char_traits<char> >&) (fpp.cc:1743)
==4092== by 0x404F49: main (foo.cc:3159)
==4092== If you believe this happened as a result of a stack
==4092== overflow in your program's main thread (unlikely but
==4092== possible), you can try to increase the size of the
==4092== main thread stack using the --main-stacksize= flag.
==4092== The main thread stack size used in this run was 8388608.
[...]
( If more output is need just ask )
I've had to change some variable names for privacy, hope this is not a problem.
Thanks for reading and have a nice day!
The first for loop is wrong:
for (std::list<unsigned int>::iterator it=myMap[iD].begin(); it != myMap[iD].end(); ++it){
myMap[iD].erase(it);
}
because the iterator it becomes invalid after each call to erase.
You could re-write it as:
for (auto it = myMap[iD].begin(); it != myMap[iD].end(); )
{
it = myMap[iD].erase(it);
}
or better yet just
myMap[iD].clear();
There seem to be other for loops in your question where you make a similar mistake - in general you need to use the pattern:
for (auto it = foo.begin(); it != foo.end(); )
{
if (some_condition)
it = foo.erase(it); // erase map entry, update iterator
else
++it; // bump iterator
}
See the example on the cppreference.com entry for std::map::erase.
My app is using boost::program_options and it's triggering an AddressSanitizer "stack-buffer-overflow" while generating an error message from an exception.
I'm not worried about the boost bug - the functionality works and this is just in the command line parsing portion of a non-production app. However I'd like to suppress the AddressSanitizer message.
ERROR: AddressSanitizer: stack-buffer-overflow on address 0x7fffe6ce7070 at pc 0x0000007406cd bp 0x7fffe6ce6fe0 sp 0x7fffe6ce6fd8
READ of size 8 at 0x7fffe6ce7070 thread T0
#0 0x7406cc in std::_Head_base<0ul, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&, false>::_M_head(std::_Head_base<0ul, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&, false>&) /frc/toolchain6/include/c++/5.3.0/tuple:142
#1 0x7406cc in _M_create_node /frc/toolchain6/include/c++/5.3.0/tuple:347
#2 0x7403fd in std::_Rb_tree_iterator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_emplace_hint_unique<std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&>, std::tuple<> >(std::_Rb_tree_const_iterator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&>&&, std::tuple<>&&) /frc/toolchain6/include/c++/5.3.0/bits/stl_tree.h:2170
#3 0xd5eff8 in boost::program_options::error_with_option_name::substitute_placeholders(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const (/home/joe/myapp_workspace/myapp/myapp-debug+0xd5eff8)
#4 0xd5c0dd in boost::program_options::error_with_option_name::what() const (/home/joe/myapp_workspace/myapp/myapp-debug+0xd5c0dd)
#5 0x58addf in main /home/joe/myapp_workspace/myapp/main.cpp:62
#6 0x7fd7e056176c in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2176c)
#7 0x436aa0 (/home/joe/myapp_workspace/myapp/myapp-debug+0x436aa0)
I've tried using the ASAN_OPTIONS suppression file method but that only seems to support a very short list of error types (such as "vptr_check" and "leak").
I don't think there's an easy way to suppress this error - Clang version of Asan has blacklisting mechanism but it's context insensitive so you'd have to disable memory checking in all usages of std::string which is highly undesirable.
One option is to use -fsanitize-recover=address compiler flag and add halt_on_error=0 to your ASAN_OPTIONS environment variable (see wiki for details and note that recovery is only supported is relatively new GCC and Clang). This will continue execution after first error. You'll then be able to examine full Asan report and select what interests you.
I am new to valgrind, so this could be missintepreting the output.
Valgrind seems to think bytes are being lost in the following lines:
if ( mysqlpp::StoreQueryResult result = query.store() ){
for ( size_t i = 0; i < result.num_rows(); ++i ){
hash = std::string( result[i]["hash"].data(), result[i]["hash"].length() );
label = std::string( result[i]["label"].data(), result[i]["label"].length() );
keywords = std::string( result[i]["keywords"].data(), result[i]["keywords"].length() );
task_raw = std::string( result[i]["task"].data(), result[i]["task"].length() );
solution_raw = std::string( result[i]["solution"].data(), result[i]["solution"].length() );
In fact it complains every time there is an std::string being constructed this way^^. I cannot figure out if this is a false-positive, if the way I'm copying is not the right way, or if the result is being leaked somehow.
Interestingly enough, it complains about another piece of code that doesn't use mysql++, but again a string constructor is used:
for ( auto & f : frames ){
for ( unsigned int i = 0; i < f.getTask().size(); i++ ){
if ( i + 1 < f.getTask().size() ){
auto t = f.getTask()[i].value();
auto n = f.getTask()[i+1].value();
pairs.push_back( std::string( t + n ) );
The line where a string is being constructed and pushed back, are possibly lost. Then a detailed output of malloc, operator new, std::string, etc are given.
The same story appears again, when copying a string from a std::istringstream:
std::copy( std::istream_iterator<std::string>(iss), std::istream_iterator<std::string>(), std::back_inserter<std::vector<std::string> >(tokens));
Are those false-positives? Does it have to do with using C++0x? Or are those legit leaks that may be originating from somewhere I have not yet detected to be leaking?
EDIT
Below is a sample of the reoccurring valgrind warning:
==18141== 9,555 bytes in 147 blocks are possibly lost in loss record 250 of 256
==18141== at 0x4C28BED: malloc (vg_replace_malloc.c:263)
==18141== by 0x7CA74C: operator new(unsigned long) (in trunk/Build/icarus)
==18141== by 0x7B4208: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator<char> const&) (in /trunk/Build/icarus)
==18141== by 0x7B5904: char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) (in trunk/Build/icarus)
==18141== by 0x7B5A7C: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, unsigned long, std::allocator<char> const&) (in trunk/Build/icarus)
==18141== by 0x610D65: Frame::mySQLloadFrameFromExample(std::string const&) (Frame.cpp:80)
==18141== by 0x6103FB: Frame::Frame(std::string const&) (Frame.cpp:5)
==18141== by 0x6054DB: Domain::mySQLloadFrames() (Domain.cpp:90)
==18141== by 0x60489D: Domain::Domain(std::string) (Domain.cpp:7)
==18141== by 0x628876: DomainController::loadDomainData() (DomainController.cpp:33)
==18141== by 0x628582: DomainController::DomainController() (DomainController.cpp:5)
==18141== by 0x627002: Controller::Controller() (Controller.cpp:26)
They are almost all identical, with the only difference being the actual line in code (i.e: the string constructor line).
The second type of warning is:
==18141== 3,562 bytes in 137 blocks are possibly lost in loss record 242 of 256
==18141== at 0x4C28BED: malloc (vg_replace_malloc.c:263)
==18141== by 0x7CA74C: operator new(unsigned long) (in trunk/Build/icarus)
==18141== by 0x7B4208: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator<char> const&) (in trunk/Build/icarus)
==18141== by 0x7B4EFA: std::string::_Rep::_M_clone(std::allocator<char> const&, unsigned long) (in trunk/Build/icarus)
==18141== by 0x7B4FDF: std::string::reserve(unsigned long) (in trunk/Build/icarus)
==18141== by 0x784F2F: std::basic_istream<char, std::char_traits<char> >& std::operator>><char, std::char_traits<char>, std::allocator<char> >(std::basic_istream<char, std::char_traits<char> >&, std::basic_string<char, std::char_traits<char>, std::allocator<char> >&) (in trunk/Build/icarus)
==18141== by 0x61208E: std::istream_iterator<std::string, char, std::char_traits<char>, long>::_M_read() (stream_iterator.h:121)
==18141== by 0x613405: std::istream_iterator<std::string, char, std::char_traits<char>, long>::operator++() (stream_iterator.h:95)
==18141== by 0x6131BC: std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > std::__copy_move<false, false, std::input_iterator_tag>::__copy_m<std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > >(std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > >) (stl_algobase.h:298)
==18141== by 0x612D09: std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > std::__copy_move_a<false, std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > >(std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > >) (stl_algobase.h:384)
==18141== by 0x6121FD: std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > std::__copy_move_a2<false, std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > >(std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > >) (stl_algobase.h:422)
==18141== by 0x611C6A: std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > std::copy<std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > > >(std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::istream_iterator<std::string, char, std::char_traits<char>, long>, std::back_insert_iterator<std::vector<std::string, std::allocator<std::string> > >) (stl_algobase.h:454)
Which seems to be related with the std::istringstream code mentioned above.