Arman Akbarian
UNIVERSITY OF BRITISH COLUMBIA
PHYSICS & ASTRONOMY DEPT.

/* Collects all T datas from all CPUS to T_s on rank 0 */

void Collect(double *T, double *T_s) {
  int MAX_MPI_SEND_COUNT = 15000;
  int rc;
  int source, dest,tag;
  MPI_Status stat;
  int loc_shape[2];
  double loc_x,loc_z;
  int i,j,shift_x,shift_z;
  int loc_phys_bdy[4];
  int left_s,right_s,top_s,bottom_s;
  int k,kk,ll;
  int indx,indx_s,ii,jj;
  int cpi;
  source = my_rank;
  dest = 0;
  tag = my_rank + 100;
  /**************MPI SEND *************/
  rc = MPI_Send(&shape[0],2,MPI_INT,dest,tag,MPI_COMM_WORLD);
  rc = MPI_Send(&x[0],1,MPI_DOUBLE,dest,tag,MPI_COMM_WORLD);
  rc = MPI_Send(&z[0],1,MPI_DOUBLE,dest,tag,MPI_COMM_WORLD);
  kk = shape[0]*shape[1]/MAX_MPI_SEND_COUNT;
  for(k=0;k<kk;k++){
    rc = MPI_Send(&T[k*MAX_MPI_SEND_COUNT],MAX_MPI_SEND_COUNT,MPI_DOUBLE,dest,tag,MPI_COMM_WORLD);
    tag++;
  }
  ll = shape[0]*shape[1] % MAX_MPI_SEND_COUNT;
  rc = MPI_Send(&T[kk*MAX_MPI_SEND_COUNT],ll,MPI_DOUBLE,dest,tag,MPI_COMM_WORLD);


  rc = MPI_Send(&phys_bdy[0],4,MPI_INT,dest,tag,MPI_COMM_WORLD);
  /**************END OF MPI SEND *************/
  if (my_rank == 0) {
      setzero(T_s,size_s);
      for(cpi=0;cpi<numprocs;cpi++) {
        source = cpi;
        tag = cpi+100;
        /************ MPI RECEIVE **********************/
         rc = MPI_Recv(&loc_shape[0], 2, MPI_INT,source, tag,
                  MPI_COMM_WORLD,&stat);
         rc = MPI_Recv(&loc_x, 1, MPI_DOUBLE,source, tag,
                    MPI_COMM_WORLD,&stat);
         rc = MPI_Recv(&loc_z, 1, MPI_DOUBLE,source, tag,
                    MPI_COMM_WORLD,&stat);
         kk = loc_shape[0]*loc_shape[1]/MAX_MPI_SEND_COUNT;
         for(k=0;k<kk;k++){
           rc = MPI_Recv(&loc_T[k*MAX_MPI_SEND_COUNT],MAX_MPI_SEND_COUNT,
                  MPI_DOUBLE,source,tag, MPI_COMM_WORLD,&stat);
           tag++;
         }
         ll = loc_shape[0]*loc_shape[1] % MAX_MPI_SEND_COUNT;
         rc = MPI_Recv(&loc_T[kk*MAX_MPI_SEND_COUNT],MAX_MPI_SEND_COUNT,
                  MPI_DOUBLE,source,tag, MPI_COMM_WORLD,&stat);

         rc = MPI_Recv(&loc_phys_bdy[0],4,MPI_INT,source,tag,
                MPI_COMM_WORLD,&stat);
        /************ END OF MPI RECEIVE **********************/
        shift_x = (int)round( (loc_x - base_bbox[0] ) / hx );
        shift_z = (int)round( (loc_z - base_bbox[2] ) / hz );

        /*Injecting to _s structure */
        for(j=0;j<loc_shape[1];j++) {
          jj = j+shift_z;
          for(i=0;i<loc_shape[0];i++) {
             ii = i+shift_x;
             indx = i+j*loc_shape[0];
             indx_s = ii+jj*Nx_s;
             T_s[indx_s] = loc_T[indx];
           }
         }



      } //End of for loop over all CPUS
  } //END of rank 0 tasks
}


last update: Wed Aug 19, 2015