MPI C++ class object use member function on slave processors - c++

Question

I am writing a code to send fixed number of rows (split along first dimension in case of 2D) and a slab (for 3D, again split along the first dimension). For example for a 100x100 I would like to send 25x100 to each of the 4 processors and in case of 3D (100x100x100) I would like to send 25x100x100 to each processors.
I have a class file and I create instance of class in the main program. My question is how 1) How can I create class objects for each process (mpi rank) and be able to use the member functions (example my_average) in the code. 2) If I create instance of class inside enclosing parenthesis the I get scope issues. 3) In the output attached below I see that send is not working. What am I missing? Is this the right way to do things.
Please help.
Pardon my lingo - c++ is not my first language.
Attached sample output.
each process gets 2
First Dimension 4
Number of processors 2
Displacement and blocklen 0 8
Displacement and blocklen 8 8
Displacement and blocklen 0 8
Displacement and blocklen 8 8
( 0.0000 + i -0.0000)( 1.0000 + i -1.1315)( 2.0000 + i -2.7556)( 3.0000 + i -3.4587)
( 0.5328 + i 0.4672)( 1.2190 + i -0.2190)( 2.0470 + i -1.0470)( 3.6789 + i -2.6789)
( 1.3586 + i 1.3207)( 2.8694 + i 0.0653)( 2.7670 + i -0.3835)( 4.0388 + i -1.5194)
( 2.4929 + i 2.1690)( 1.1037 + i 1.9654)( 2.1604 + i 0.9465)( 4.5891 + i -0.5297)
end of 2D matrix
rank=0 Rows : 2 Width : 4 local_n0 size 2
( 0.0000 + i -0.0000)( 1.0000 + i -1.1315)( 2.0000 + i -2.7556)( 3.0000 + i -3.4587)
( 0.5328 + i 0.4672)( 1.2190 + i -0.2190)( 2.0470 + i -1.0470)( 3.6789 + i -2.6789)
rank=1 Rows : 2 Width : 4 local_n0 size 2
( 0.0000 + i 0.0000)( 0.0000 + i 0.0000)( 0.0000 + i 0.0000)( 0.0000 + i 0.0000)
( 0.0000 + i 0.0000)( 0.0000 + i 0.0000)( 0.0000 + i 0.0000)( 0.0000 + i 0.0000)
main program
#include <fftw3-mpi.h>
#include <cstdlib>
#include <iostream>
#include <iomanip> // needed for setfill
#include "cmplx_2d.hh"
#include "random.hh"
using namespace std;
int main(int argc, char **argv){
const int N0 = 4, N1 = 4;
int i,j;
int rank,size;
double rx;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
int local_n0=N0/size;
if(rank==0){
cout << " each process gets " << local_n0 << endl;
cout << " First Dimension " << N0 << endl;
cout << " Number of processors " << size << endl;
}
/* compute start and size of rows */
int n_rows[size];
int send_counts[size];
int displs[size];
displs[0]=0;
for (i = 0; i < size; ++i) {
n_rows[i]=N0/size;
if(i<N0%size){
n_rows[i]=n_rows[i]+1;
}
send_counts[i]=n_rows[i]*N1;
if(i>0){
displs[i]=displs[i-1]+send_counts[i-1];
}
cout << " Displacement and blocklen " << displs[i] << " " << send_counts[i] << endl;
}
int rows=n_rows[rank];
/* create 2d complex objects */
cmplx_2d arr_cmplx(N0,N1);
if(rank==0) {
/* initialize data -- random number */
for (i = 0; i < N0; ++i) {
for (j = 0; j < N1; ++j){
rx =((double)rand()/(double)RAND_MAX);
arr_cmplx(i,j)=complex<double>(rx*i+j,i-j-rx);
printf("(%8.4f + i%8.4f)", real(arr_cmplx(i,j)), imag(arr_cmplx(i,j)));
}
printf(" \n ");
}
printf(" \n end of 2D matrix \n \n ");
} // end of master
if(rank==0) {
MPI_Scatterv(&arr_cmplx(0,0),send_counts,displs,MPI_DOUBLE_COMPLEX,MPI_IN_PLACE,0,MPI_DOUBLE_COMPLEX,0,MPI_COMM_WORLD);
} else {
/* I should allocate local array on each processor to receive data but if I do it here I get scope error */
//cmplx_2d arr_cmplx(rows,N1);
MPI_Scatterv(NULL,send_counts,displs,MPI_DOUBLE_COMPLEX,&arr_cmplx(0,0),rows*N1,MPI_DOUBLE_COMPLEX,0,MPI_COMM_WORLD);
}
//printing, one proc at a time
if(rank>0){
MPI_Status status;
MPI_Recv(NULL,0,MPI_DOUBLE_COMPLEX,rank-1,0,MPI_COMM_WORLD,&status);
}
cout<<"rank="<< rank<<" Rows : "<<rows<<" Width : "<<N1<< " local_n0 size " << local_n0 << endl;
for(i=0; i<rows; i++)
{
for(j=0; j<N1; j++)
printf("(%8.4f + i%8.4f)", real(arr_cmplx(i,j)), imag(arr_cmplx(i,j)));
cout<<endl;
}
if(rank<size-1){
MPI_Send(NULL,0,MPI_DOUBLE_COMPLEX,rank+1,0,MPI_COMM_WORLD);
}
MPI_Finalize();
return 0;
}
header file
#include <cassert> // for assert()
#include <complex>
using namespace std;
class cmplx_2d
{
private:
int nx;
int ny;
complex<double>** array_2d;
public:
// constructor
cmplx_2d(int x, int y) {
array_2d = new complex<double>*[x];
for(int i(0); i < x; ++i) {
array_2d[i] = new complex<double>[y];
for(int j(0); j < y; ++j) {
array_2d[i][j]= complex<double>(0.,0.);
}}
nx=x;
ny=y;
}
// deconstructor
~cmplx_2d() {
delete [] array_2d;
}
// overload () operator
complex<double>& operator()(const int i, const int j) const;
// prototype of overload constructor - just a dummy function
void my_avg( const cmplx_2d& in, int* ir, int* il, int* jr, int* jl, const int flag );
int getNx() const {
return nx;
}
int getNy() const {
return ny;
}
}; // end of class
header file
#include <iostream>
#include "cmplx_2d.hh"
#include "main_header.hh"
// overload constructor to - just a dummy for now
void cmplx_2d::my_avg( const cmplx_2d& in, int* ir, int* il, int* jr, int* jl, const int flag ) {
nx=in.getNx();
ny=in.getNy();
if ( flag == DX )
{
// derivative with respect to x
for(int i(0); i < nx; ++i) {
for(int j(0); j < ny; ++j) {
array_2d[i][j] = 0.5*(in(ir[i],j)+in(il[i],j));
}}
}
else
{
std::cout << "NOT a valid flag for derivative!" << std::endl;
}
} // end
// overloading of () parenthesis operator
complex<double>& cmplx_2d::operator()(const int i, const int j) const
{
assert(i >= 0 && i < nx);
assert(j >= 0 && j < ny);
return array_2d[i][j];
}
main header
const double PI=4.0*atan(1.0);
const double TWO_PI=2.0*PI;
enum flag {DX, DY, LAP};