OpenMP slower than serial codes - c++

Question

I am trying to do the parallelization of a serial preconditioned conjugate gradient solver codes for 3D fire simulation using OpenMP (Intel compiler). But the performance seems not to be improved.
The grid dimension is 79x81x79 and the solver can converge after 565 iterations. The serial codes cost 3.39 seconds and the OpenMP version needs 3.86 seconds on Intel i7 2600 (OS: openSUSE 13.1).
Please help me to check the codes. Thanks a lot.
// preconditioned conjugate gradient solver ...
void PCGSSolver::solveNew(const Array3D<double>& sn, const Array3D<double>& ae, const Array3D<double>&aw,
const Array3D<double>& as, const Array3D<double>& an, const Array3D<double>&at, const Array3D<double>&ab,
const Array3D<double>& ap, Array3D<double>& ptmp){
std::size_t dimX=sn.getDimI();
std::size_t dimY=sn.getDimJ();
std::size_t dimZ=sn.getDimK();
Array3D<double> p1(dimX,dimY,dimZ,0.0);
Array3D<double> res(dimX,dimY,dimZ,0.0);
Array3D<double> d(dimX,dimY,dimZ,0.0);
Array3D<double> ain(dimX,dimY,dimZ,0.0);
double tiny=1.0e-30;
#pragma omp parallel
{
//Jacobi preconditioner
#pragma omp for nowait
for(std::size_t k=1;k<dimZ-1; k++){
for(std::size_t j=1; j<dimY-1; j++){
for(std::size_t i=1; i<dimX-1; i++){
d(i,j,k)=1./ap(i,j,k);
}
}
}
#pragma omp for nowait
for(std::size_t k=1;k<dimZ-1; k++){
for(std::size_t j=1; j<dimY-1; j++){
for(std::size_t i=1; i<dimX-1; i++){
res(i,j,k)=ae(i,j,k)*ptmp(i+1,j,k) + aw(i,j,k)*ptmp(i-1,j,k)+an(i,j,k)*ptmp(i,j+1,k)+as(i,j,k)*ptmp(i,j-1,k)+
at(i,j,k)*ptmp(i,j,k+1)+ab(i,j,k)*ptmp(i,j,k-1)+sn(i,j,k)-ap(i,j,k)*ptmp(i,j,k);
}
}
}
}
double big =1.0e+30;
double s1old=big;
//start iteration
for(std::size_t intswp=0; intswp<this->nswpvr; intswp++){
double alpha=0.0;
double bbeta=0.0;
double s1=0.0;
double s2=0.0;
double testir=0.0;
#pragma omp parallel
{
#pragma omp for reduction(+:s1)
for(std::size_t k=1;k<dimZ-1; k++){
for(std::size_t j=1; j<dimY-1; j++){
for(std::size_t i=1; i<dimX-1; i++){
ain(i,j,k)=res(i,j,k)*d(i,j,k);
s1+=(res(i,j,k)*ain(i,j,k));
}
}
}
#pragma omp single
{
bbeta=s1/(s1old+tiny);
}
#pragma omp for
for(std::size_t k=1;k<dimZ-1; k++){
for(std::size_t j=1; j<dimY-1; j++){
for(std::size_t i=1; i<dimX-1; i++){
p1(i,j,k)=ain(i,j,k)+bbeta*p1(i,j,k);
}
}
}
#pragma omp for reduction(+:s2)
for(std::size_t k=1;k<dimZ-1; k++){
for(std::size_t j=1; j<dimY-1; j++){
for(std::size_t i=1; i<dimX-1; i++){
ain(i,j,k)=ap(i,j,k)*p1(i,j,k)-ae(i,j,k)*p1(i+1,j,k)-aw(i,j,k)*p1(i-1,j,k)-
an(i,j,k)*p1(i,j+1,k)-as(i,j,k)*p1(i,j-1,k)-
at(i,j,k)*p1(i,j,k+1)-ab(i,j,k)*p1(i,j,k-1);
s2+=(p1(i,j,k)*ain(i,j,k));
}
}
}
#pragma omp single
{
alpha=s1/(s2+tiny);
}
#pragma omp for reduction(+:testir)
for(std::size_t k=1;k<dimZ-1; k++){
for(std::size_t j=1; j<dimY-1; j++){
for(std::size_t i=1; i<dimX-1; i++){
ptmp(i,j,k)=ptmp(i,j,k)+alpha*p1(i,j,k);
res(i,j,k)=res(i,j,k)-alpha*ain(i,j,k);
testir+=fabs(res(i,j,k));
}
}
}
}//==openmp region end
s1old=s1;
//test stop criteria
if(testir < ccvar){
std::cout<<"PCGS solver coverage at "<<(intswp+1)<<" iterations!"<<std::scientific<<testir<<std::endl;
return;
}
}
std::cout<<"PCGS solver can not coverage "<<std::endl;
}
The Array3D is a my 3 dimension array class.
#ifndef ARRAY3D_H
#define ARRAY3D_H
#include <vector>
#include <algorithm>
template<typename T> class Array3D
{
public:
typedef T value_type;
Array3D(){
dim_i=dim_j=dim_k=0;
dim_ij=0;
}
Array3D(std::size_t size_i, std::size_t size_j, std::size_t size_k){
this->resize(size_i,size_j,size_k);
}
Array3D(std::size_t size_i, std::size_t size_j, std::size_t size_k,const value_type& defaultValue){
this->resize(size_i,size_j,size_k,defaultValue);
}
virtual ~Array3D(){}
std::size_t getDimI()const{
return this->dim_i;
}
std::size_t getDimJ()const{
return this->dim_j;
}
std::size_t getDimK()const{
return this->dim_k;
}
//check if valid indices
bool checkIndices(std::size_t i, std::size_t j, std::size_t k){
return (i<this->dim_i ) && (j<this->dim_j) && (k<this->dim_k);
}
void resize(std::size_t size_i, std::size_t size_j, std::size_t size_k,const value_type& defaultValue){
this->resize(size_i,size_j,size_k);
this->fillValue(defaultValue);
}
//resize the array. The data will be ereased.
void resize(std::size_t size_i, std::size_t size_j, std::size_t size_k){
this->dim_i=size_i;
this->dim_j=size_j;
this->dim_k=size_k;
this->dim_ij=this->dim_i*this->dim_j;
std::size_t totalSize=this->dim_i*this->dim_j*this->dim_k;
this->data.resize(totalSize);
}
std::size_t size()const{
return this->data.size();
}
void fillValue(const value_type& defaultValue){
std::fill(this->data.begin(),this->data.end(),defaultValue);
}
value_type minValue()const{
return *(std::min_element(data.begin(),data.end()));
}
value_type maxValue()const{
return *(std::max_element(data.begin(),data.end()));
}
//Fill the array value using the sum of two array
void setValueSum(const Array3D& array1, const Array3D& array2){
size_t minSize=std::min(std::min(array1.data.size(),array2.data.size()),this->data.size());
for(size_t i=0; i<minSize; i++)
this->data[i]=array1.data[i]+array2.data[i];
}
void clear(){
dim_i=dim_j=dim_k=0;
dim_ij=0;
this->data.clear();
}
//get value reference at (i,j,k) or (x,y,z) or (u,v,w)...
const value_type& operator () (std::size_t i, std::size_t j, std::size_t k )const{
return this->data.at(this->calIndex(i,j,k));
}
value_type& operator ()(std::size_t i, std::size_t j, std::size_t k ){
return this->data.at(this->calIndex(i,j,k));
}
//access the raw data by 1D index
const value_type& operator [] (std::size_t i )const{
return this->data.at(i);
}
value_type& operator [](std::size_t i ){
return this->data.at(i);
}
std::vector<value_type>* rawData(){
return &(data);
}
private:
inline std::size_t calIndex(std::size_t i, std::size_t j, std::size_t k )const{
return k*this->dim_ij+j*this->dim_i+i;
}
private:
//dimension of array (i,j,k)(x,y,z)(u,v,w)...
std::size_t dim_i, dim_j, dim_k;
//raw data, order is I-J-K
std::vector<value_type> data;
//dim_i*dim_j
std::size_t dim_ij;
};
#endif // ARRAY3D_H
I measure the time using a Timer class codes downloaded from internet.
timer.start();
PCGSSolver solver;
solver.setTolerance(this->ccvar);
solver.setIteNum(this->nswpp);
solver.solveNew(sn,ae,aw,as,an,at,ab,ap,ptmp);
timer.stop();
std::cout<<"PCGS time:"<<timer.getElapsedTimeInSec()<<"sec"<<std::endl;
Timer.h
//////////////////////////////////////////////////////////////////////////////
// Timer.h
// =======
// High Resolution Timer.
// This timer is able to measure the elapsed time with 1 micro-second accuracy
// in both Windows, Linux and Unix system
//
// AUTHOR: Song Ho Ahn (song.ahn#gmail.com)
// CREATED: 2003-01-13
// UPDATED: 2006-01-13
//
// Copyright (c) 2003 Song Ho Ahn
//////////////////////////////////////////////////////////////////////////////
#ifndef TIMER_H_DEF
#define TIMER_H_DEF
#ifdef WIN32 // Windows system specific
#include <windows.h>
#else // Unix based system specific
#include <sys/time.h>
#endif
class Timer
{
public:
Timer(); // default constructor
~Timer(); // default destructor
void start(); // start timer
void stop(); // stop the timer
double getElapsedTime(); // get elapsed time in second
double getElapsedTimeInSec(); // get elapsed time in second (same as getElapsedTime)
double getElapsedTimeInMilliSec(); // get elapsed time in milli-second
double getElapsedTimeInMicroSec(); // get elapsed time in micro-second
protected:
private:
double startTimeInMicroSec; // starting time in micro-second
double endTimeInMicroSec; // ending time in micro-second
int stopped; // stop flag
#ifdef WIN32
LARGE_INTEGER frequency; // ticks per second
LARGE_INTEGER startCount; //
LARGE_INTEGER endCount; //
#else
timeval startCount; //
timeval endCount; //
#endif
};
#endif // TIMER_H_DEF
Timer.cpp
//////////////////////////////////////////////////////////////////////////////
// Timer.cpp
// =========
// High Resolution Timer.
// This timer is able to measure the elapsed time with 1 micro-second accuracy
// in both Windows, Linux and Unix system
//
// AUTHOR: Song Ho Ahn (song.ahn#gmail.com)
// CREATED: 2003-01-13
// UPDATED: 2006-01-13
//
// Copyright (c) 2003 Song Ho Ahn
//////////////////////////////////////////////////////////////////////////////
#include "Timer.h"
#include <stdlib.h>
///////////////////////////////////////////////////////////////////////////////
// constructor
///////////////////////////////////////////////////////////////////////////////
Timer::Timer()
{
#ifdef WIN32
QueryPerformanceFrequency(&frequency);
startCount.QuadPart = 0;
endCount.QuadPart = 0;
#else
startCount.tv_sec = startCount.tv_usec = 0;
endCount.tv_sec = endCount.tv_usec = 0;
#endif
stopped = 0;
startTimeInMicroSec = 0;
endTimeInMicroSec = 0;
}
///////////////////////////////////////////////////////////////////////////////
// distructor
///////////////////////////////////////////////////////////////////////////////
Timer::~Timer()
{
}
///////////////////////////////////////////////////////////////////////////////
// start timer.
// startCount will be set at this point.
///////////////////////////////////////////////////////////////////////////////
void Timer::start()
{
stopped = 0; // reset stop flag
#ifdef WIN32
QueryPerformanceCounter(&startCount);
#else
gettimeofday(&startCount, NULL);
#endif
}
///////////////////////////////////////////////////////////////////////////////
// stop the timer.
// endCount will be set at this point.
///////////////////////////////////////////////////////////////////////////////
void Timer::stop()
{
stopped = 1; // set timer stopped flag
#ifdef WIN32
QueryPerformanceCounter(&endCount);
#else
gettimeofday(&endCount, NULL);
#endif
}
///////////////////////////////////////////////////////////////////////////////
// compute elapsed time in micro-second resolution.
// other getElapsedTime will call this first, then convert to correspond resolution.
///////////////////////////////////////////////////////////////////////////////
double Timer::getElapsedTimeInMicroSec()
{
#ifdef WIN32
if(!stopped)
QueryPerformanceCounter(&endCount);
startTimeInMicroSec = startCount.QuadPart * (1000000.0 / frequency.QuadPart);
endTimeInMicroSec = endCount.QuadPart * (1000000.0 / frequency.QuadPart);
#else
if(!stopped)
gettimeofday(&endCount, NULL);
startTimeInMicroSec = (startCount.tv_sec * 1000000.0) + startCount.tv_usec;
endTimeInMicroSec = (endCount.tv_sec * 1000000.0) + endCount.tv_usec;
#endif
return endTimeInMicroSec - startTimeInMicroSec;
}
///////////////////////////////////////////////////////////////////////////////
// divide elapsedTimeInMicroSec by 1000
///////////////////////////////////////////////////////////////////////////////
double Timer::getElapsedTimeInMilliSec()
{
return this->getElapsedTimeInMicroSec() * 0.001;
}
///////////////////////////////////////////////////////////////////////////////
// divide elapsedTimeInMicroSec by 1000000
///////////////////////////////////////////////////////////////////////////////
double Timer::getElapsedTimeInSec()
{
return this->getElapsedTimeInMicroSec() * 0.000001;
}
///////////////////////////////////////////////////////////////////////////////
// same as getElapsedTimeInSec()
///////////////////////////////////////////////////////////////////////////////
double Timer::getElapsedTime()
{
return this->getElapsedTimeInSec();
}

Answer 1

A quick glance over your code shows a few areas where you can improve the performance. I'll leave the implementation up to you.
OMP Parallel For
Firstly its generally cheaper to use
#pragma omp parallel for
for (...) {
...
}
versus
#pragma omp parallel
{
#pragma omp for
for (...) {
...
}
}
Not by much but there is a slight improvement. See [1], the graphic at the end.
OMP SINGLE
The key benefit of using #pragma omp parallel for in this case is that it allows us to remove the #pragma omp single directive. When your program encounters a #pragma omp single directive every thread waits here until the others are finished processing their chunk of work. This could lead to a situation where several of your threads finish early and have to wait on another to finish until they can proceed.
Use of #pragma omp single and #pragma omp barrier is strongly discouraged in high performing parallelised code.
Collapsing Loops (The Hard Way)
The next area you need to look at is collapsing your loops. The following
#pragma omp parallel for
for (int k = 0; k < o; ++k) {
for (int j = 0; j < m; ++j) {
for (int i = 0; i < n; ++i) {
...
}
}
}
will generally parallelise the outer loop for (int k = ...) but run the inner loops in serial on each thread. You can achieve parallelisation of the entire loop by unravelling them like
#pragma omp parallel for
for (int l = 0; l < o*m*n; ++l) {
int i = l % n;
int j = (l / n) % m;
int k = ((l / n) / m) % o;
...
}
In most of your loops you can simply use l and the overloaded [] operator. Most Conjugate Gradient solvers will only need the l index and not the i, j and k indices as they operate on vectors. The only time when i, j and k are needed is when you are computing A*x (or A'*x). This change will increase the level of parallelisation in your code and should provide noticeable improvements.
Collapsing Loops (The Easy Way)
It should be mentioned that OpenMP as of Version 3.0 supports the collapse(n) clause which can be used to tell the compiler to automatically collapse the for() loops as I've described above. An example of this is
#pragma omp parallel for collapse(3)
for (int k = 0; k < o; ++k) {
for (int j = 0; j < m; ++j) {
for (int i = 0; i < n; ++i) {
...
}
}
}
which will cause the compiler to form a single for() loop and then parallelise it.
Reduction Clause
Lastly, and probably the most costly element in your code is the reduction() clause. Edit: I incorrectly previously mentioned this could be removed after collapsing the loops in my haste to finish the answer.
Source [1]

Answer 2

I don't know precisely why the OpenMP parallelisation doesn't make the code faster, but what's obvious is that you've got all you loops in the wrong order.
So, first thing first, start by swapping the i and k loops in your code and I'm sure you'll see a dramatic performance boost. Then you can have a look at OpenMP.