HARM
harm and utilities
 All Data Structures Files Functions Variables Typedefs Macros Pages
boundmpiint.c
Go to the documentation of this file.
1 #include "decs.h"
2 
9 
10 int bound_mpi_int_dir(int boundstage, int finalstep, int whichdir, int boundvartype, PFTYPE (*prim)[NSTORE2][NSTORE3][NUMPFLAGS])
11 {
12  int dir;
13  static MPI_Request requests[COMPDIM * 2 * 2];
14  static int didpostrecvs[COMPDIM*2]={0};
15 
16  int dirstart,dirfinish;
17  if(whichdir==-1 || whichdir==1){ dirstart=X1UP; dirfinish=X1DN;}
18  if(whichdir==-2 || whichdir==2){ dirstart=X2UP; dirfinish=X2DN;}
19  if(whichdir==-3 || whichdir==3){ dirstart=X3UP; dirfinish=X3DN;}
20 
21 
22 
23 
24 
25 
27  //
28  // pre-post recv's
29  //
30  // OPTMARK: Could avoid dir-dependent MPI calls in step_ch.c (currently true!) and post all recv's for all dirs at once.
31  // OPTMARK: Or setup MPIFLOWCONTROL==2 or SUPERMPI
32  //
34  if(whichdir==-1 || whichdir==-2 || whichdir==-3){
35  if((boundstage==STAGEM1)||(boundstage==STAGE0)){
36  for(dir=dirstart;dir<=dirfinish;dir++){
37  if(dirgenset[boundvartype][dir][DIRIF]){
38  recvonly_int(dir,boundvartype,workbc_int,requests);
39  didpostrecvs[dir]=1;
40  }
41  }
42  }
43  }
44  else if(whichdir==1 || whichdir==2 || whichdir==3){
45 
47  //
48  // per-whichdir (1,2,3) conditionals for bounding. Assume whichdir=1 done first, then 2, then 3, so that corners are done correctly as done with normal bounds.
49  //
51 
53  //
54  // x or y or z -dir
55  // once dir=0,1(X1UP,X1DN) is done, so can start 2,3(X2UP,X2DN)
56  //
58  if((boundstage==STAGEM1)||(boundstage==STAGE0&&whichdir==1 || boundstage==STAGE2&&whichdir==2 || boundstage==STAGE4&&whichdir==3)){
59  for(dir=dirstart;dir<=dirfinish;dir++){
60  if(dirgenset[boundvartype][dir][DIRIF]){
61  if(didpostrecvs[dir]==0){
62  dualfprintf(fail_file,"Did not post recv and tried to already pack: dir=%d\n",dir);
63  myexit(234525155);
64  }
65  pack_int(dir,boundvartype,prim,workbc_int);
66  }
67  }
68  for(dir=dirstart;dir<=dirfinish;dir++) if(dirgenset[boundvartype][dir][DIRIF]) sendonly_int(dir,boundvartype,workbc_int,requests);
69  }
70  if((boundstage==STAGEM1)||(boundstage==STAGE1&&whichdir==1 || boundstage==STAGE3&&whichdir==2 || boundstage==STAGE5&&whichdir==3)){
71  for(dir=dirstart;dir<=dirfinish;dir++) if(dirgenset[boundvartype][dir][DIRIF]){
72  recvwait(dir,requests);
73  didpostrecvs[dir]=0; // done with recv's
74  }
75  for(dir=dirstart;dir<=dirfinish;dir++) if(dirgenset[boundvartype][dir][DIRIF]) unpack_int(dir,boundvartype,workbc_int,prim);
76  for(dir=dirstart;dir<=dirfinish;dir++) if(dirgenset[boundvartype][dir][DIRIF]) sendwait(dir,requests);
77  }
78  }
79  else{
80  dualfprintf(fail_file,"No such whichdir=%d in boundmpiint.c\n",whichdir);
81  myexit(1986290387);
82  }
83 
84 
85 
86  // now corner zones will be filled correctly
87  // GODMARK: If made fixup_utoprim() and check_solution() not use corner zones could bound all directions at once -- probably not important performance hit
88 
89 
90  return(0);
91 
92 }
93 // end function
94 
95 
96 #define PACKLOOP_INT(i,j,k,istart,istop,jstart,jstop,kstart,kstop,di,dj,dk) FBOUNDLOOP(pl) SUPERGENLOOP(i,j,k,istart,istop,jstart,jstop,kstart,kstop,di,dj,dk)
97 
99 void pack_int(int dir, int boundvartype,PFTYPE (*prim)[NSTORE2][NSTORE3][NUMPFLAGS],PFTYPE (*workbc_int)[COMPDIM * 2][NUMPFLAGSBOUND * NBIGBND * NBIGSM])
100 {
101  // dir=direction sending
102  int i,j,k;
103  int pl,pliter;
104  int bci;
105 
106 
107  bci=0;
108  PACKLOOP_INT(i,j,k
109  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPSTART1]
110  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPSTOP1]
111  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPSTART2]
112  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPSTOP2]
113  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPSTART3]
114  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPSTOP3]
115  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPDIR1]
116  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPDIR2]
117  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRPDIR3]
118  ){
119  /*
120  if(bci>=dirgenset[boundvartype][dir][DIRSIZE]){
121  dualfprintf(fail_file,"pack memory leak: bci: %d dirgenset[%d][DIRSIZE]: %d\n",bci,dirgenset[boundvartype][dir][DIRSIZE]);
122  myexit(10);
123  }
124  */
125  workbc_int[PACK][dir][bci++] = MACP0A1(prim,i,j,k,pl);
126  }
127 }
128 
129 
130 void recvonly_int(int dir, int boundvartype,PFTYPE (*workbc_int)[COMPDIM * 2][NUMPFLAGSBOUND * NBIGBND * NBIGSM],MPI_Request *requests)
131 {
132  MPI_Irecv(workbc_int[UNPACK][dir],
133  dirgenset[boundvartype][dir][DIRSIZE],
134  MPI_PFTYPE,
135  MPIid[dirgenset[boundvartype][dir][DIROTHER]],
136  TAGSTARTBOUNDMPIINT + dirgenset[boundvartype][dir][DIRTAGR],
137  MPI_COMM_GRMHD,
138  &requests[dir*2+REQRECV]);
139 
140 
141 }
142 
143 
144 void sendonly_int(int dir, int boundvartype,PFTYPE (*workbc_int)[COMPDIM * 2][NUMPFLAGSBOUND * NBIGBND * NBIGSM],MPI_Request *requests)
145 {
146 
147 
148  if(MPIFLOWCONTROL==1){
149  int nothingsend=0;
150  int nothingrecv=0;
151  int maxtag = numprocs*COMPDIM*2;
152 
153  MPI_Sendrecv(
154  &nothingsend,0,MPI_INT,
155  MPIid[dirgenset[boundvartype][dir][DIROTHER]],
156  TAGSTARTBOUNDMPIINT + maxtag + dirgenset[boundvartype][dir][DIRTAGS],
157 
158  &nothingrecv,0,MPI_INT,
159  MPIid[dirgenset[boundvartype][dir][DIROTHER]],
160  TAGSTARTBOUNDMPIINT + maxtag + dirgenset[boundvartype][dir][DIRTAGR],
161 
162  MPI_COMM_GRMHD,MPI_STATUS_IGNORE);
163  } // end if doing FLOWCONTROL
164 
165 
166 
167  MPI_Isend(workbc_int[PACK][dir],
168  dirgenset[boundvartype][dir][DIRSIZE],
169  MPI_PFTYPE,
170  MPIid[dirgenset[boundvartype][dir][DIROTHER]],
171  TAGSTARTBOUNDMPIINT + dirgenset[boundvartype][dir][DIRTAGS],
172  MPI_COMM_GRMHD,
173  &requests[dir*2+REQSEND]);
174 
175 }
176 
177 
178 void unpack_int(int dir, int boundvartype,PFTYPE (*workbc_int)[COMPDIM * 2][NUMPFLAGSBOUND * NBIGBND * NBIGSM],PFTYPE (*prim)[NSTORE2][NSTORE3][NUMPFLAGS])
179 {
180  // dir is direction receiving from
181  int i,j,k;
182  int pl,pliter;
183  int bci;
184 
185  bci=0;
186  PACKLOOP_INT(i,j,k
187  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTART1]
188  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTOP1]
189  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTART2]
190  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTOP2]
191  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTART3]
192  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTOP3]
193  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUDIR1]
194  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUDIR2]
195  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUDIR3]
196  ){
197  MACP0A1(prim,i,j,k,pl)=workbc_int[UNPACK][dir][bci++];
198  }
199 }
200 
201 
202 
203 
204 
205 
206 
207 
208 
209 
210 
213 int bound_mpi_int_fakeutoprimmpiinconsisent(int boundstage, int finalstep, int fakedir, int boundvartype, PFTYPE (*prim)[NSTORE2][NSTORE3][NUMPFLAGS], int fakevalue)
214 {
215  int dir;
216 
217 
218  if(fakedir==+1){ // no need for fakedir==-1
219  // only need to "unpack"
220  for(dir=X1UP;dir<=X1DN;dir++) if(dirgenset[boundvartype][dir][DIRIF]) unpack_int_fakeutoprimmpiinconsisent(dir,boundvartype,workbc_int,prim,fakevalue);
221  for(dir=X2UP;dir<=X2DN;dir++) if(dirgenset[boundvartype][dir][DIRIF]) unpack_int_fakeutoprimmpiinconsisent(dir,boundvartype,workbc_int,prim,fakevalue);
222  for(dir=X3UP;dir<=X3DN;dir++) if(dirgenset[boundvartype][dir][DIRIF]) unpack_int_fakeutoprimmpiinconsisent(dir,boundvartype,workbc_int,prim,fakevalue);
223  }
224 
225  return(0);
226 
227 }
228 
229 
231 void unpack_int_fakeutoprimmpiinconsisent(int dir, int boundvartype,PFTYPE (*workbc_int)[COMPDIM * 2][NUMPFLAGSBOUND * NBIGBND * NBIGSM],PFTYPE (*prim)[NSTORE2][NSTORE3][NUMPFLAGS], int fakevalue)
232 {
233  // dir is direction receiving from
234  int i,j,k;
235  int pl,pliter;
236  int bci;
237 
238  bci=0;
239  PACKLOOP_INT(i,j,k
240  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTART1]
241  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTOP1]
242  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTART2]
243  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTOP2]
244  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTART3]
245  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUSTOP3]
246  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUDIR1]
247  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUDIR2]
248  ,dirloopset[boundvartype][dir][primgridpos[boundvartype][dir][pl]][DIRUDIR3]
249  ){
250  MACP0A1(prim,i,j,k,pl)=fakevalue;
251  }
252 }
253 
254 
255 
256 
257 
259 int bound_mpi_int(int boundstage, int finalstep, int fakedir, int boundvartype, PFTYPE (*prim)[NSTORE2][NSTORE3][NUMPFLAGS])
260 {
261  int bound_mpi_int_dir(int boundstage, int finalstep, int whichdir, int boundvartype, PFTYPE (*prim)[NSTORE2][NSTORE3][NUMPFLAGS]);
262  int whichdir;
263 
264  // separate pre-post recv and intermediate user bound and then final other MPI stuff -- implies TAG space is globally separated for these MPI calls and any MPI stuff done by user!
265  if(fakedir==-1){
266  whichdir=-1; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim); // pre-post MPI recv's
267  whichdir=-2; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim); // pre-post MPI recv's
268  whichdir=-3; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim); // pre-post MPI recv's
269  }
270  else if(fakedir==1){
271  whichdir=1; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim);
272  whichdir=2; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim);
273  whichdir=3; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim);
274  }
275  else if(fakedir==0){
276  whichdir=-1; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim); // pre-post MPI recv's
277  whichdir=1; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim);
278  whichdir=-2; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim); // pre-post MPI recv's
279  whichdir=2; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim);
280  whichdir=-3; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim); // pre-post MPI recv's
281  whichdir=3; bound_mpi_int_dir(boundstage, finalstep, whichdir, boundvartype, prim);
282  }
283 
284  return(0);
285 }
286 
287 
Generated on Fri May 20 2016 15:52:32 for HARM by   doxygen 1.8.3.1