swapcontext failing in many-to-many thread implementation - c++

Question

I am tasked with implementing a many-to-many thread manager in C++. I've got most of it more or less down, but I'm having serious problems with swapcontext in my uthread_yield() method. Here's the code
/*
* uthread.cpp
*
* Created on: Oct 12, 2016
* Author: michael
*/
#include "uthread.h"
#include <semaphore.h>
#include <pthread.h>
#ifndef STDIO_H_
#include <stdio.h>
#endif
namespace std {
/*
* Initializes all the variables and allocates memory when needed
*/
int uthread::maxThreads;
int uthread::currentThreads;
pthread_mutex_t uthread::mapMutex;
pthread_mutex_t uthread::qMutex;
pthread_mutex_t uthread::threadMutex;
map<int,UserThread*>* uthread::threadMap;
priority_queue<UserThread*>* uthread::threadQueue;
void uthread::uthread_init(int numKernelThreads) {
pthread_mutex_t tester;
pthread_mutex_init(&tester,NULL);
uthread::maxThreads=numKernelThreads;
uthread::currentThreads=0;
pthread_mutex_init(&threadMutex,NULL);
pthread_mutex_init(&qMutex,NULL);
pthread_mutex_init(&mapMutex,NULL);
threadQueue= new priority_queue<UserThread*>;
threadMap=new map<int,UserThread*>;
}
int uthread::uthread_create(void (* func)( )) {
//Create ucontext to be used in in
ucontext_t* ucp=(ucontext_t*)malloc(sizeof(ucontext_t));
getcontext(ucp);
ucp->uc_stack.ss_sp=(void*)malloc(16384);
ucp->uc_stack.ss_size=16384;
makecontext(ucp, func, 0); //make the context for a thread running func
//Create UserThread
time_t currTime;
time(&currTime);
UserThread* newThread=new UserThread(ucp,currTime);
//Thread Creation Logic
pthread_mutex_lock(&threadMutex);
if (currentThreads>=maxThreads) {
pthread_mutex_unlock(&threadMutex);
pthread_mutex_lock(&qMutex);
threadQueue->push(newThread);
pthread_mutex_unlock(&qMutex);
return 0;
}
else {
int (*execute)(void *)= (int (*)(void *)) func;
int tid=clone(execute,ucp->uc_stack.ss_sp,CLONE_VM|CLONE_FILES,NULL);
if (tid==-1) { //clone failed
pthread_mutex_unlock(&threadMutex);
return -1;
}
currentThreads++;
pthread_mutex_unlock(&threadMutex);
/*
* Map tid -> UserThread in thread map
*/
threadMap->insert(pair<int,UserThread*>(tid,newThread));
pthread_mutex_unlock(&mapMutex);
return 0;
}
return -1;
}
void uthread::uthread_exit() {
/*
* Get the corresponding UserThread object from the map
*/
printf("Start Exit \n");
int threadID=syscall(SYS_gettid) ;
pthread_mutex_lock(&mapMutex);
if (threadMap->find(threadID)==threadMap->end()) { //Cannot find map;
pthread_mutex_lock(&threadMutex);
currentThreads--;
pthread_mutex_unlock(&threadMutex);
exit(0);
}
printf("Getting Curr Thread\n");
UserThread* currThread= threadMap->at(threadID);
pthread_mutex_unlock(&mapMutex);
pthread_mutex_lock(&qMutex);
if (threadQueue->empty()) { //No items on queue, delete memory references and exit
printf("Queue is Empty");
pthread_mutex_unlock(&qMutex);
pthread_mutex_lock(&mapMutex);
threadMap->erase(threadID);
pthread_mutex_unlock(&mapMutex);
pthread_mutex_lock(&threadMutex);
currentThreads--;
pthread_mutex_unlock(&threadMutex);
delete currThread;
exit(0);
}
else { //Remove and delete memory reference to old thread, set context to new thread
printf("Swapping Queue\n");
UserThread* newThread=threadQueue->top();
threadQueue->pop();
pthread_mutex_unlock(&qMutex);
pthread_mutex_lock(&mapMutex);
threadMap->insert(pair<int,UserThread*>(threadID,newThread)); //Update Map
pthread_mutex_unlock(&mapMutex);
printf("Deleting Current Thread\n");
delete currThread;
printf("Setting Context\n");
setcontext(newThread->ucp);
printf("set context failed\n");
}
}
void uthread::uthread_yield() {
printf("Start Yield \n");
int threadID=syscall(SYS_gettid) ;
pthread_mutex_lock(&mapMutex);
UserThread* currThread= threadMap->at(threadID);
pthread_mutex_unlock(&mapMutex);
pthread_mutex_lock(&qMutex);
if (threadQueue->empty()) {
printf("Queue is empty\n");
pthread_mutex_unlock(&qMutex);
return;
}
else {
printf("Queue Not Empty\n");
currThread->updateRuntime(time(NULL)); //updates run time account for time it's been on thread
UserThread* highestPriority=threadQueue->top();
if (highestPriority->getRunTime()>currThread->getRunTime()) { //highestPriority is lower priority than currently running thread
printf("lowest runtime is running\n");
pthread_mutex_unlock(&qMutex);
return;
}
else {
printf("SwapContext\n");
threadQueue->pop();
threadQueue->push(currThread);
pthread_mutex_unlock(&qMutex);
pthread_mutex_lock(&mapMutex);
threadMap->insert(pair<int,UserThread*>(threadID,highestPriority)); //Update Map reference
pthread_mutex_unlock(&mapMutex);
//Swaps contexts
swapcontext(currThread->ucp,highestPriority->ucp);
printf("Swapcontext Failed\n");
}
}
}
int uthread::startThread(void* arg ) {
printf("Thread Cloned\n");
pthread_mutex_lock(&mapMutex);
int threadID=syscall(SYS_gettid) ;
UserThread* currThread= threadMap->at(threadID);
pthread_mutex_unlock(&mapMutex);
setcontext(currThread->ucp);
return 0;
}
}
And this is the code of my corresponding UserThread object:
/*
* UserThread.cpp
*
* Created on: Oct 12, 2016
* Author: michael
*/
#include "UserThread.h"
/*
* Constructor. UCP is taken in as well as start time
* Run time initialized to 0
*
*/
UserThread::UserThread(ucontext_t *ucp,time_t st) {
this->ucp=ucp;
this->startTime=(time_t*)malloc(sizeof(time_t));
this->runTime=(double*)malloc(sizeof(double));
*startTime=st;
*runTime=0;
}
/**
* Deconstructor
*/
UserThread::~UserThread() {
//free(ucp->uc_stack.ss_sp);
//free(ucp);
free(startTime);
free(runTime);
}
/*
* adds the running time in seconds (as a double) to the current running time. Also updates the start time
*/
void UserThread::updateRuntime(time_t currTime) {
double diffTime=difftime(currTime,*startTime);
*runTime=*runTime+diffTime;
*startTime=currTime;
}
/*
* Just Updates the start time
*/
void UserThread::updateStartTime(time_t newTime) {
*startTime=newTime;
}
/*
* getter
*/
double UserThread::getRunTime() {
double rTime=*runTime;
return rTime;
}
/*
* getter
*/
time_t UserThread::getStartTime() {
return *startTime;
}
/*
* THIS IS REVERSED ON PURPOSE. C++ runs a maximum priority queue by default
* by overloading the < operator backwards, that isn't an issue. Sketchy? Yes
* Also functional
*/
bool UserThread::operator <(UserThread* t2) {
return this->getRunTime() > t2->getRunTime();
}
uthread_yield() will correctly work once for each thread, then fail. Any idea why this is? I've stared at this code for hours and at this point I'm out of ideas.

Answer 1

It's not actually failing: it's just printing your failure message. Your yield implementation finishes with:
swapcontext(currThread->ucp, highestPriority->ucp);
printf("Swapcontext Failed\n");
So the yielding thread swaps away after swapcontext(), which is fine - but when that thread is later swapped back to, it will return from swapcontext() and unconditionally execute the printf(). You need:
if (swapcontext(currThread->ucp, highestPriority->ucp) == -1) {
printf("Swapcontext Failed\n");
/* You need to fix up your queue and map here to account for the
* failure to context switch, and then probably loop back and look
* for another candidate thread to swap to. */
}
I also noticed that your uthread_create() function accesses the map and unlocks mapMutex without locking the mutex first.
You're mixing pthreads functions with the bare clone() syscall, which is unsupported. Use pthread_create() / pthread_exit() for managing the underlying threads. One way to do this is by having thread created by pthread_create() start at a scheduling function that pulls the thread to run from your queue:
void* uthread::newThread(void* arg)
{
pthread_mutex_lock(&qMutex);
if (threadQueue->empty()) {
printf("No thread to start.\n");
return NULL;
}
UserThread* highestPriority = threadQueue->top();
threadQueue->pop();
pthread_mutex_unlock(&qMutex);
int threadID = syscall(SYS_gettid);
pthread_mutex_lock(&mapMutex);
threadMap->insert(pair<int,UserThread*>(threadID,highestPriority)); //Update Map reference
pthread_mutex_unlock(&mapMutex);
setcontext(highestPriority->ucp);
printf("setcontext() Failed\n");
return NULL;
}
...then you can simplify uthread_create() by always pushing the new user thread onto the queue, and only conditionally creating the underlying thread:
// ... start of uthread_create() up to creating new UserThread ...
pthread_mutex_lock(&qMutex);
threadQueue->push(newThread);
pthread_mutex_unlock(&qMutex);
//Thread Creation Logic
pthread_mutex_lock(&threadMutex);
if (currentThreads < maxThreads) {
pthread_t new_pthread;
if (pthread_create(*new_pthread, NULL, uthread::newThread, NULL) != 0) {
printf("New pthread creation failed.\n");
} else {
currentThreads++;
}
}
pthread_mutex_unlock(&threadMutex);
return 0;
By the way, it seems that you're using the threadMap just as a way to implement thread-local-storage: you could instead use the built-in pthreads thread-local-storage API (pthread_key_create() / pthread_setspecific() / pthread_getspecific()).