MPI C - Gather 2d Array Segments into One Global Array - multidimensional-array

Question

I am trying to print a dynamically allocated 2d array from my master process after receiving all its components from all other processes. By components I mean subarrays, or blocks.
I have made the code generic to the number of processes. The following diagram will help you see how the blocks are arranged in the complete array. Each block is handled by one process. Just for here though, let's assume that i run the program using 12 processes (natively i have 8 cores), using the command:
mpiexec -n 12 ./gather2dArray
This is the diagram, which targets specifically the 12 process scenario:
The answer by Jonathan in this question helped me a great deal, but unfortunately i have not been able to fully implement what i want.
I first create the blocks into each process, which i name them grid. Every array is a dynamically allocated 2d array. I also create the global array (universe) to be visible only by the master process (#0).
Finally i have to use MPI_Gatherv(...) to assemble all the subarrays into the global array. Then i proceed to display the local arrays and the global array.
When i run the program with the command above i get Segmentation fault when i reach the MPI_Gatherv(...) function. I can't figure out what i do incorrectly. I have provided complete code (heavily commented) below:
EDIT
I have fixed some wrongs in the code. Now MPI_Gatherv() is somewhat successful. I am able to print the entire first row of the global array correctly (i check the individual elements of the processes and they always match). But when i reach the second row some hieroglyphics appear and finally a segmentation fault. I haven't been able to figure out what is wrong there. Still looking into it..
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <time.h>
void print2dCharArray(char** array, int rows, int columns);
int main(int argc, char** argv)
{
int master = 0, np, rank;
char version[10];
char processorName[20];
int strLen[10];
// Initialize MPI environment
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &np);
if (np != 12) { MPI_Abort(MPI_COMM_WORLD,1); }
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
// We need a different seed for each process
srand(time(0) ^ (rank * 33 / 4));
int nDims = 2; // array dimensions
int rows = 4, columns = 6; // rows and columns of each block
int prows = 3, pcolumns = 4; // rows and columns of blocks. Each block is handled by 1 process
char** grid = malloc(rows * sizeof(char*));
for (int i = 0; i < rows; i++)
grid[i] = malloc(columns * sizeof(char));
char** universe = NULL; // Global array
char* recvPtr; // Pointer to start of Global array
int Rows = rows * prows; // Global array rows
int Columns = columns * pcolumns; // Global array columns
int sizes[2]; // No of elements in each dimension of the whole array
int subSizes[2]; // No of elements in each dimension of the subarray
int startCoords[2]; // Starting coordinates of each subarray
MPI_Datatype recvBlock, recvMagicBlock;
if (rank == master){ // For the master's eyes only
universe = malloc(Rows * sizeof(char*));
for (int i = 0; i < Rows; i++)
universe[i] = malloc(Columns * sizeof(char));
// Create a subarray (a rectangular block) datatype from a regular, 2d array
sizes[0] = Rows;
sizes[1] = Columns;
subSizes[0] = rows;
subSizes[1] = columns;
startCoords[0] = 0;
startCoords[1] = 0;
MPI_Type_create_subarray(nDims, sizes, subSizes, startCoords, MPI_ORDER_C, MPI_CHAR, &recvBlock);
// Now modify the newly created datatype to fit our needs, by specifying
// (lower bound remains the same = 0)
// - new extent
// The new region / block will now "change" sooner, as soon as we reach a region of elements
// occupied by a new block, ie. every: (columns) * sizeof(elementType) =
MPI_Type_create_resized(recvBlock, 0, columns * sizeof(char), &recvMagicBlock);
MPI_Type_commit(&recvMagicBlock);
recvPtr = &universe[0][0];
}
// populate arrays
for (int y = 0; y < rows; y++){
for (int x = 0; x < columns; x++){
if (( (double) rand() / RAND_MAX) <= density)
grid[y][x] = '#';
else
grid[y][x] = '.';
}
}
// display local array
for (int i = 0; i < np; i++){
if (i == rank) {
printf("\n[Rank] of [total]: No%d of %d\n", rank, np);
print2dCharArray(grid, rows, columns);
}
MPI_Barrier(MPI_COMM_WORLD);
}
/* MPI_Gathering.. */
int recvCounts[np], displacements[np];
// recvCounts: how many chunks of data each process has -- in units of blocks here --
for (int i = 0; i < np; i++)
recvCounts[i] = 1;
// prows * pcolumns = np
// displacements: displacement relative to global buffer (universe) at which to place the
// incoming data block from process i -- in block extents! --
int index = 0;
for (int p_row = 0; p_row < prows; p_row++)
for (int p_column = 0; p_column < pcolumns; p_column++)
displacements[index++] = p_column + p_row * (rows * pcolumns);
// MPI_Gatherv(...) is a collective routine
// Gather the local arrays to the global array in the master process
// send type: MPI_CHAR (a char)
// recv type: recvMagicBlock (a block)
MPI_Gatherv(&grid[0][0], rows * columns, MPI_CHAR, //: parameters relevant to sender
recvPtr, recvCounts, displacements, recvMagicBlock, master, //: parameters relevant to receiver
MPI_COMM_WORLD);
// display global array
MPI_Barrier(MPI_COMM_WORLD);
if (rank == master){
printf("\n---Global Array---\n");
print2dCharArray(universe, Rows, Columns);
}
MPI_Finalize();
return 0;
}
void print2dCharArray(char** array, int rows, int columns)
{
int i, j;
for (i = 0; i < rows; i++){
for (j = 0; j < columns; j++){
printf("%c ", array[i][j]);
}
printf("\n");
}
fflush(stdout);
}
The following is the output I'm getting. No matter what i try, I cannot get past this. As you can see the first line of the global array is printed properly using the first 4 blocks of the 4 processes. When jumping to next line we get hieroglyphics..
hostname#User:~/mpi$ mpiexec -n 12 ./gather2darray
MPICH Version: 3User
Processor name: User
[Rank] of [total]: No0 of 12
. . # . . #
# . # # # .
. . . # # .
. . # . . .
[Rank] of [total]: No1 of 12
. . # # . .
. . . . # #
. # . . # .
. . # . . .
[Rank] of [total]: No2 of 12
. # # # . #
. # . . . .
# # # . . .
. . . # # .
[Rank] of [total]: No3 of 12
. . # # # #
. . # # . .
# . # . # .
. . . # . .
[Rank] of [total]: No4 of 12
. # . . . #
# . # . # .
# . . . . .
# . . . . .
[Rank] of [total]: No5 of 12
# # . # # .
# . . # # .
. . . . # .
. # # . . .
[Rank] of [total]: No6 of 12
. . # # . #
. . # . # .
# . . . . .
. . . # # #
[Rank] of [total]: No7 of 12
# # . # # .
. # # . . .
. . . . . #
. . . # # .
[Rank] of [total]: No8 of 12
. # . . . .
# . # . # .
. . . # . #
# . # # # .
[Rank] of [total]: No9 of 12
. . . . . #
. . # . . .
. . # . . #
. . # # . .
[Rank] of [total]: No10 of 12
. . . . # .
# . . . . .
. . # # . .
. . . # . #
[Rank] of [total]: No11 of 12
. # . . # .
. # . # # .
. . . # . .
. # . # . #
---Global Array---
. . # . . # . . # # . . . # # # . # . . # # # #
� � < * � � e { � � � � � �
J
*** Error in `./gather2darray': double free or corruption (out): 0x0000000001e4c050 ***
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
*** stack smashing detected ***: ./gather2darray terminated
===================================================================================
= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES
= PID 10979 RUNNING AT User
= EXIT CODE: 139
= CLEANING UP REMAINING PROCESSES
= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES
===================================================================================
YOUR APPLICATION TERMINATED WITH THE EXIT STRING: Segmentation fault (signal 11)
This typically refers to a problem with your application.
Please see the FAQ page for debugging suggestions
Help will be very appreciated. Thanks in advance.

Answer 1

Your code is almost correct, you just forgotten an MPI important principle. When you are using an array on MPI functions, MPI assumes that your array memory is allocate continuously. So you have to change your 2 dims arrays allocations.
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <time.h>
void print2dCharArray(char** array, int rows, int columns);
int main(int argc, char** argv)
{
int master = 0, np, rank;
char version[10];
char processorName[20];
int strLen[10];
// Initialize MPI environment
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &np);
if (np != 12) { MPI_Abort(MPI_COMM_WORLD,1); }
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
// We need a different seed for each process
srand(time(0) ^ (rank * 33 / 4));
int nDims = 2; // array dimensions
int rows = 4, columns = 6; // rows and columns of each block
int prows = 3, pcolumns = 4; // rows and columns of blocks. Each block is handled by 1 process
char* pre_grid = (char*) malloc(rows * columns * sizeof(char));
char** grid = (char**) malloc(rows * sizeof(char*));
for (int i = 0; i < rows; i++)
grid[i] = &(pre_grid[i * columns]);
char** universe = NULL; // Global array
char* pre_universe = NULL;
char* recvPtr; // Pointer to start of Global array
int Rows = rows * prows; // Global array rows
int Columns = columns * pcolumns; // Global array columns
int sizes[2]; // No of elements in each dimension of the whole array
int subSizes[2]; // No of elements in each dimension of the subarray
int startCoords[2]; // Starting coordinates of each subarray
MPI_Datatype recvBlock, recvMagicBlock;
if (rank == master){ // For the master's eyes only
/* universe = malloc(Rows * sizeof(char*));*/
/* for (int i = 0; i < Rows; i++)*/
/* universe[i] = malloc(Columns * sizeof(char));*/
pre_universe = (char*) malloc(Rows * Columns * sizeof(char));
universe = (char**) malloc(Rows * sizeof(char*));
for (int i = 0; i < Rows; i++) {
universe[i] = &(pre_universe[i * Columns]);
}
// Create a subarray (a rectangular block) datatype from a regular, 2d array
sizes[0] = Rows;
sizes[1] = Columns;
subSizes[0] = rows;
subSizes[1] = columns;
startCoords[0] = 0;
startCoords[1] = 0;
MPI_Type_create_subarray(nDims, sizes, subSizes, startCoords, MPI_ORDER_C, MPI_CHAR, &recvBlock);
// Now modify the newly created datatype to fit our needs, by specifying
// (lower bound remains the same = 0)
// - new extent
// The new region / block will now "change" sooner, as soon as we reach a region of elements
// occupied by a new block, ie. every: (columns) * sizeof(elementType) =
MPI_Type_create_resized(recvBlock, 0, columns * sizeof(char), &recvMagicBlock);
MPI_Type_commit(&recvMagicBlock);
recvPtr = &universe[0][0];
}
// populate arrays
for (int y = 0; y < rows; y++){
for (int x = 0; x < columns; x++){
grid[y][x] = rank + 65;
}
}
// display local array
for (int i = 0; i < np; i++){
if (i == rank) {
printf("\n[Rank] of [total]: No%d of %d\n", rank, np);
print2dCharArray(grid, rows, columns);
}
MPI_Barrier(MPI_COMM_WORLD);
}
/* MPI_Gathering.. */
int recvCounts[np], displacements[np];
// recvCounts: how many chunks of data each process has -- in units of blocks here --
for (int i = 0; i < np; i++)
recvCounts[i] = 1;
// prows * pcolumns = np
// displacements: displacement relative to global buffer (universe) at which to place the
// incoming data block from process i -- in block extents! --
int index = 0;
for (int p_row = 0; p_row < prows; p_row++)
for (int p_column = 0; p_column < pcolumns; p_column++)
displacements[index++] = p_column + p_row * (rows * pcolumns);
// MPI_Gatherv(...) is a collective routine
// Gather the local arrays to the global array in the master process
// send type: MPI_CHAR (a char)
// recv type: recvMagicBlock (a block)
MPI_Gatherv(&grid[0][0], rows * columns, MPI_CHAR, //: parameters relevant to sender
recvPtr, recvCounts, displacements, recvMagicBlock, master, //: parameters relevant to receiver
MPI_COMM_WORLD);
// display global array
MPI_Barrier(MPI_COMM_WORLD);
if (rank == master){
printf("\n---Global Array---\n");
print2dCharArray(universe, Rows, Columns);
}
free(grid[0]);
free(grid);
if (rank == master) {
free(universe[0]);
free(universe);
MPI_Type_free(&recvMagicBlock);
MPI_Type_free(&recvBlock);
}
MPI_Finalize();
return 0;
}
void print2dCharArray(char** array, int rows, int columns)
{
int i, j;
for (i = 0; i < rows; i++){
for (j = 0; j < columns; j++){
printf("%c ", array[i][j]);
}
printf("\n");
}
fflush(stdout);
}
Output:
---Global Array---
A A A A A A B B B B B B C C C C C C D D D D D D
A A A A A A B B B B B B C C C C C C D D D D D D
A A A A A A B B B B B B C C C C C C D D D D D D
A A A A A A B B B B B B C C C C C C D D D D D D
E E E E E E F F F F F F G G G G G G H H H H H H
E E E E E E F F F F F F G G G G G G H H H H H H
E E E E E E F F F F F F G G G G G G H H H H H H
E E E E E E F F F F F F G G G G G G H H H H H H
I I I I I I J J J J J J K K K K K K L L L L L L
I I I I I I J J J J J J K K K K K K L L L L L L
I I I I I I J J J J J J K K K K K K L L L L L L
I I I I I I J J J J J J K K K K K K L L L L L L