HARM
harm and utilities
 All Data Structures Files Functions Variables Typedefs Macros Pages
mpi_grmhd_grray_liaison.c
Go to the documentation of this file.
1 
7 #include "decs.h"
8 
9 
10 
11 
12 
13 #if(USEMPIANY)
14 
15 //
16 // Include these declarations and functions only if any code that uses this function is doing MPI
17 //
19 
20 
23 static void get_processtypelist(int processtype, MPI_Comm localcomm, int **local_processtypelistlocal, int *local_sizeproclist);
24 static void liaison_init_mpi_processtypes(MPI_Comm localcomm, int **processtypelistlocal_world, int *local_sizeproclist);
25 static void grmhd_init_mpi_processtypes(MPI_Comm localcomm, int **processtypelistlocal_world, int *local_sizeproclist);
26 static void grray_init_mpi_processtypes(MPI_Comm localcomm, int **processtypelistlocal_world, int *local_sizeproclist);
27 
28 
30 void liaison_init_mpi_processtypes(MPI_Comm localcomm, int **processtypelistlocal_world, int *local_sizeproclist)
31 {
32  get_processtypelist(BINARYLIAISONTYPE,localcomm,processtypelistlocal_world,local_sizeproclist);
33 
34 }
35 
37 void grmhd_init_mpi_processtypes(MPI_Comm localcomm, int **processtypelistlocal_world, int *local_sizeproclist)
38 {
39  get_processtypelist(BINARYGRMHDTYPE,localcomm,processtypelistlocal_world,local_sizeproclist);
40 
41 }
42 
44 void grray_init_mpi_processtypes(MPI_Comm localcomm, int **processtypelistlocal_world, int *local_sizeproclist)
45 {
46  get_processtypelist(BINARYGRRAYTYPE,localcomm,processtypelistlocal_world,local_sizeproclist);
47 
48 }
49 
50 
51 
54 void get_processtypelist(int processtype, MPI_Comm localcomm, int **processtypelistlocal, int *sizeproclistlocal)
55 {
56  int i;
57 
58 
59  // may already have this, but get again to keep code modular
60  MPI_Comm_size(localcomm, sizeproclistlocal);
61  stderrfprintf("processtype=%d sizeproclistlocal=%d\n",processtype,*sizeproclistlocal); fflush(stderr);
62 
63  // return;
64 
65  // allocate processor list
66  *processtypelistlocal=(int*) malloc((*sizeproclistlocal)*sizeof(int));
67  if(*processtypelistlocal==NULL){
68  stderrfprintf("Could not allocate processtypelistlocal\n");
69  exit(1);
70  }
71  stderrfprintf("After malloc of processtypelistlocal: %d %d\n",processtype,myid_world); fflush(stderr);
72 
73  // Gather process type list
74  MPI_Allgather(&processtype,1,MPI_INT,*processtypelistlocal,1,MPI_INT,localcomm);
75  stderrfprintf("After MPI_Allgather: %d\n",myid_world); fflush(stderr);
76  // MPI_Barrier(localcomm);
77 
78 
79  // stderrfprintf("processtype=%d sizeproclistlocal0=%d\n",processtype,*sizeproclistlocal);
80  // for(i=0;i<*sizeproclistlocal;i++){
81  // stderrfprintf("it0[%d]=%d\n",i,(*processtypelistlocal)[i]); fflush(stderr);
82  // }
83 
84 
85 }
86 
87 
88 #endif // end if any code is doing MPI
89 
90 
91 
92 
93 
94 
95 
96 
97 
98 
99 #if(USEMPILIAISON)
100 
101 //
102 // Below functions are used if really doing liaison method with MPI
103 //
105 
106 
108 static void init_MPI_group_grmhd_grray_liaison(
109  int *processtypelistlocal_world,int sizeproclistlocal_world,
110  MPI_Group *MPI_GROUP_WORLD,
111  MPI_Group *MPI_GROUP_GRMHD_LIAISON, MPI_Comm *MPI_COMM_GRMHD_LIAISON,
112  MPI_Group *MPI_GROUP_GRRAY_LIAISON, MPI_Comm *MPI_COMM_GRRAY_LIAISON);
113 
114 
115 
116 
120 void liaison_init_mpi_liaisonmode_globalset(void)
121 {
122  void liaison_init_mpi_liaisonmode(
123  int **processtypelistlocal_world,int *sizeproclistlocal_world,
124  int **processtypelistlocal_grmhd_liaison,int *sizeproclistlocal_grmhd_liaison,
125  int **processtypelistlocal_grray_liaison,int *sizeproclistlocal_grray_liaison,
126  int **processtypelistlocal_grmhd,int *sizeproclistlocal_grmhd,
127  int **processtypelistlocal_grray,int *sizeproclistlocal_grray,
128  int **processtypelistlocal_liaison_from_grmhd,int *sizeproclistlocal_liaison_from_grmhd,
129  int **processtypelistlocal_liaison_from_grray,int *sizeproclistlocal_liaison_from_grray,
130  MPI_Group *MPI_GROUP_WORLD,
131  MPI_Group *MPI_GROUP_GRMHD_LIAISON, MPI_Comm *MPI_COMM_GRMHD_LIAISON,
132  MPI_Group *MPI_GROUP_GRRAY_LIAISON, MPI_Comm *MPI_COMM_GRRAY_LIAISON,
133  MPI_Group *MPI_GROUP_GRMHD, MPI_Comm *MPI_COMM_GRMHD,
134  MPI_Group *MPI_GROUP_GRRAY, MPI_Comm *MPI_COMM_GRRAY,
135  MPI_Group *MPI_GROUP_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LIAISON_FROM_GRMHD,
136  MPI_Group *MPI_GROUP_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LIAISON_FROM_GRRAY);
137 
138 
139  liaison_init_mpi_liaisonmode(
140  &processtypelist_world,&sizeproclist_world,
141  &processtypelist_grmhd_liaison,&sizeproclist_grmhd_liaison,
142  &processtypelist_grray_liaison,&sizeproclist_grray_liaison,
143  &processtypelist_grmhd,&sizeproclist_grmhd,
144  &processtypelist_grray,&sizeproclist_grray,
145  &processtypelist_liaison_from_grmhd,&sizeproclist_grmhd,
146  &processtypelist_liaison_from_grray,&sizeproclist_grray,
147  &MPI_GROUP_WORLD,
148  &MPI_GROUP_GRMHD_LIAISON, &MPI_COMM_GRMHD_LIAISON,
149  &MPI_GROUP_GRRAY_LIAISON, &MPI_COMM_GRRAY_LIAISON,
150  &MPI_GROUP_GRMHD, &MPI_COMM_GRMHD,
151  &MPI_GROUP_GRRAY, &MPI_COMM_GRRAY,
152  &MPI_GROUP_LIAISON_FROM_GRMHD, &MPI_COMM_LIAISON_FROM_GRMHD,
153  &MPI_GROUP_LIAISON_FROM_GRRAY, &MPI_COMM_LIAISON_FROM_GRRAY);
154 }
155 
156 
157 
158 
159 
163 void grmhd_init_mpi_liaisonmode_globalset(void)
164 {
165  void grmhd_init_mpi_liaisonmode(
166  int **processtypelistlocal_world,int *sizeproclistlocal_world,
167  int **processtypelistlocal_grmhd_liaison,int *sizeproclistlocal_grmhd_liaison,
168  int **processtypelistlocal_grray_liaison,int *sizeproclistlocal_grray_liaison,
169  int **processtypelistlocal_grmhd,int *sizeproclistlocal_grmhd,
170  int **processtypelistlocal_liaison_from_grmhd,int *sizeproclistlocal_liaison_from_grmhd,
171  MPI_Group *MPI_GROUP_WORLD,
172  MPI_Group *MPI_GROUP_GRMHD_LIAISON, MPI_Comm *MPI_COMM_GRMHD_LIAISON,
173  MPI_Group *MPI_GROUP_GRRAY_LIAISON, MPI_Comm *MPI_COMM_GRRAY_LIAISON,
174  MPI_Group *MPI_GROUP_GRMHD, MPI_Comm *MPI_COMM_GRMHD,
175  MPI_Group *MPI_GROUP_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LIAISON_FROM_GRMHD);
176 
177  grmhd_init_mpi_liaisonmode(
178  &processtypelist_world,&sizeproclist_world,
179  &processtypelist_grmhd_liaison,&sizeproclist_grmhd_liaison,
180  &processtypelist_grray_liaison,&sizeproclist_grray_liaison,
181  &processtypelist_grmhd,&sizeproclist_grmhd,
182  &processtypelist_liaison_from_grmhd,&sizeproclist_grmhd,
183  &MPI_GROUP_WORLD,
184  &MPI_GROUP_GRMHD_LIAISON, &MPI_COMM_GRMHD_LIAISON,
185  &MPI_GROUP_GRRAY_LIAISON, &MPI_COMM_GRRAY_LIAISON,
186  &MPI_GROUP_GRMHD, &MPI_COMM_GRMHD,
187  &MPI_GROUP_LIAISON_FROM_GRMHD, &MPI_COMM_LIAISON_FROM_GRMHD);
188 }
189 
190 
191 
192 
193 
194 
195 
199 void grray_init_mpi_liaisonmode_globalset(void)
200 {
201  void grray_init_mpi_liaisonmode(
202  int **processtypelistlocal_world,int *sizeproclistlocal_world,
203  int **processtypelistlocal_grmhd_liaison,int *sizeproclistlocal_grmhd_liaison,
204  int **processtypelistlocal_grray_liaison,int *sizeproclistlocal_grray_liaison,
205  int **processtypelistlocal_grray,int *sizeproclistlocal_grray,
206  int **processtypelistlocal_liaison_from_grray,int *sizeproclistlocal_liaison_from_grray,
207  MPI_Group *MPI_GROUP_WORLD,
208  MPI_Group *MPI_GROUP_GRMHD_LIAISON, MPI_Comm *MPI_COMM_GRMHD_LIAISON,
209  MPI_Group *MPI_GROUP_GRRAY_LIAISON, MPI_Comm *MPI_COMM_GRRAY_LIAISON,
210  MPI_Group *MPI_GROUP_GRRAY, MPI_Comm *MPI_COMM_GRRAY,
211  MPI_Group *MPI_GROUP_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LIAISON_FROM_GRRAY);
212 
213  grray_init_mpi_liaisonmode(
214  &processtypelist_world,&sizeproclist_world,
215  &processtypelist_grmhd_liaison,&sizeproclist_grmhd_liaison,
216  &processtypelist_grray_liaison,&sizeproclist_grray_liaison,
217  &processtypelist_grray,&sizeproclist_grray,
218  &processtypelist_liaison_from_grray,&sizeproclist_liaison_from_grray,
219  &MPI_GROUP_WORLD,
220  &MPI_GROUP_GRMHD_LIAISON, &MPI_COMM_GRMHD_LIAISON,
221  &MPI_GROUP_GRRAY_LIAISON, &MPI_COMM_GRRAY_LIAISON,
222  &MPI_GROUP_GRRAY, &MPI_COMM_GRRAY,
223  &MPI_GROUP_LIAISON_FROM_GRRAY, &MPI_COMM_LIAISON_FROM_GRRAY);
224 
225 }
226 
227 
228 
229 
230 
231 
233 void liaison_init_mpi_liaisonmode(
234  int **processtypelistlocal_world,int *sizeproclistlocal_world,
235  int **processtypelistlocal_grmhd_liaison,int *sizeproclistlocal_grmhd_liaison,
236  int **processtypelistlocal_grray_liaison,int *sizeproclistlocal_grray_liaison,
237  int **processtypelistlocal_grmhd,int *sizeproclistlocal_grmhd,
238  int **processtypelistlocal_grray,int *sizeproclistlocal_grray,
239  int **processtypelistlocal_liaison_from_grmhd,int *sizeproclistlocal_liaison_from_grmhd,
240  int **processtypelistlocal_liaison_from_grray,int *sizeproclistlocal_liaison_from_grray,
241  MPI_Group *MPI_GROUP_LOCAL_WORLD,
242  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
243  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
244  MPI_Group *MPI_GROUP_LOCAL_GRMHD, MPI_Comm *MPI_COMM_LOCAL_GRMHD,
245  MPI_Group *MPI_GROUP_LOCAL_GRRAY, MPI_Comm *MPI_COMM_LOCAL_GRRAY,
246  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRMHD,
247  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRRAY)
248 {
249  void init_MPI_group_grmhd_grray_liaison_split(
250  int *processtypelistlocal_grmhd_liaison,int sizeproclistlocal_grmhd_liaison,
251  int *processtypelistlocal_grray_liaison,int sizeproclistlocal_grray_liaison,
252  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
253  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
254  MPI_Group *MPI_GROUP_LOCAL_GRMHD, MPI_Comm *MPI_COMM_LOCAL_GRMHD,
255  MPI_Group *MPI_GROUP_LOCAL_GRRAY, MPI_Comm *MPI_COMM_LOCAL_GRRAY,
256  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRMHD,
257  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRRAY);
258 
259 #if(USEMPILIAISON)
260 
261 
262  // get processtypes
263  liaison_init_mpi_processtypes(MPI_COMM_WORLD,processtypelistlocal_world,sizeproclistlocal_world);
264 
265  // global group sets:
266  init_MPI_group_grmhd_grray_liaison(
267  *processtypelistlocal_world,*sizeproclistlocal_world,
268  MPI_GROUP_LOCAL_WORLD,
269  MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_COMM_LOCAL_GRMHD_LIAISON,
270  MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_COMM_LOCAL_GRRAY_LIAISON);
271 
272  // get processtypes for newly created subgroups
273  // 1) Get GRMHD+LIAISON
274  liaison_init_mpi_processtypes(*MPI_COMM_LOCAL_GRMHD_LIAISON,processtypelistlocal_grmhd_liaison,sizeproclistlocal_grmhd_liaison);
275  // 2) Get GRRAY+LIAISON
276  liaison_init_mpi_processtypes(*MPI_COMM_LOCAL_GRRAY_LIAISON,processtypelistlocal_grray_liaison,sizeproclistlocal_grray_liaison);
277 
278  // liaison split from liaison+grmhd and liaison+grray groups
279  init_MPI_group_grmhd_grray_liaison_split(
280  *processtypelistlocal_grmhd_liaison,*sizeproclistlocal_grmhd_liaison,
281  *processtypelistlocal_grray_liaison,*sizeproclistlocal_grray_liaison,
282  MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_COMM_LOCAL_GRMHD_LIAISON,
283  MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_COMM_LOCAL_GRRAY_LIAISON,
284  MPI_GROUP_LOCAL_GRMHD, MPI_COMM_LOCAL_GRMHD,
285  MPI_GROUP_LOCAL_GRRAY, MPI_COMM_LOCAL_GRRAY,
286  MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_COMM_LOCAL_LIAISON_FROM_GRMHD,
287  MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_COMM_LOCAL_LIAISON_FROM_GRRAY);
288 
289  // ensure NULL
290  if(*MPI_COMM_LOCAL_GRMHD!=MPI_COMM_NULL){
291  stderrfprintf("MPI_COMM_LOCAL_GRMHD still has members of LIAISON!\n"); fflush(stderr);
292  exit(1);
293  }
294  // ensure NULL
295  if(*MPI_COMM_LOCAL_GRRAY!=MPI_COMM_NULL){
296  stderrfprintf("MPI_COMM_LOCAL_GRRAY still has members of LIAISON!\n"); fflush(stderr);
297  exit(1);
298  }
299 
300  // get processtypes for newly created subgroups
301  // 1) Do LIAISON from GRMHD+LIAISON
302  liaison_init_mpi_processtypes(*MPI_COMM_LOCAL_LIAISON_FROM_GRMHD,processtypelistlocal_liaison_from_grmhd,sizeproclistlocal_liaison_from_grmhd);
303 
304  // 2) Do LIAISON from GRRAY+LIAISON
305  liaison_init_mpi_processtypes(*MPI_COMM_LOCAL_LIAISON_FROM_GRRAY,processtypelistlocal_liaison_from_grray,sizeproclistlocal_liaison_from_grray);
306 
307 
308 #endif
309 
310 
311 }
312 
313 
314 
315 
316 
317 
319 void grmhd_init_mpi_liaisonmode(
320  int **processtypelistlocal_world,int *sizeproclistlocal_world,
321  int **processtypelistlocal_grmhd_liaison,int *sizeproclistlocal_grmhd_liaison,
322  int **processtypelistlocal_grray_liaison,int *sizeproclistlocal_grray_liaison,
323  int **processtypelistlocal_grmhd,int *sizeproclistlocal_grmhd,
324  int **processtypelistlocal_liaison_from_grmhd,int *sizeproclistlocal_liaison_from_grmhd,
325  MPI_Group *MPI_GROUP_LOCAL_WORLD,
326  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
327  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
328  MPI_Group *MPI_GROUP_LOCAL_GRMHD, MPI_Comm *MPI_COMM_LOCAL_GRMHD,
329  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRMHD)
330 {
331  void init_MPI_group_grmhd_liaison_split(
332  int *processtypelistlocal_grmhd_liaison,int sizeproclistlocal_grmhd_liaison,
333  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
334  MPI_Group *MPI_GROUP_LOCAL_GRMHD, MPI_Comm *MPI_COMM_LOCAL_GRMHD,
335  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRMHD);
336 
337 
338  stderrfprintf("USEMPILIAISON=%d\n",USEMPILIAISON);
339 
340 #if(USEMPILIAISON)
341 
342 
343 
344  // get processtypes
345  grmhd_init_mpi_processtypes(MPI_COMM_WORLD,processtypelistlocal_world,sizeproclistlocal_world);
346 
347 
348  // global group sets:
349  init_MPI_group_grmhd_grray_liaison(
350  *processtypelistlocal_world,*sizeproclistlocal_world,
351  MPI_GROUP_LOCAL_WORLD,
352  MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_COMM_LOCAL_GRMHD_LIAISON,
353  MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_COMM_LOCAL_GRRAY_LIAISON);
354 
355  // ensure really NULL
356  if(*MPI_COMM_LOCAL_GRRAY_LIAISON!=MPI_COMM_NULL){
357  stderrfprintf("GRRAY_LIAISON still has members of GRMHD!\n"); fflush(stderr);
358  exit(1);
359  }
360 
361  // get processtypes for newly created subgroups
362  // 1) Get GRMHD+LIAISON proc type list
363  grmhd_init_mpi_processtypes(*MPI_COMM_LOCAL_GRMHD_LIAISON,processtypelistlocal_grmhd_liaison,sizeproclistlocal_grmhd_liaison);
364 
365  // liaison split from liaison+grmhd group
366  init_MPI_group_grmhd_liaison_split(
367  *processtypelistlocal_grmhd_liaison,*sizeproclistlocal_grmhd_liaison,
368  MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_COMM_LOCAL_GRMHD_LIAISON,
369  MPI_GROUP_LOCAL_GRMHD, MPI_COMM_LOCAL_GRMHD,
370  MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_COMM_LOCAL_LIAISON_FROM_GRMHD);
371 
372  // ensure really NULL
373  if(*MPI_COMM_LOCAL_LIAISON_FROM_GRMHD!=MPI_COMM_NULL){
374  stderrfprintf("MPI_COMM_LOCAL_LIAISON_FROM_GRMHD still has members of GRMHD!\n"); fflush(stderr);
375  exit(1);
376  }
377 
378  // get processtypes for newly created subgroups
379  // 1) Get GRMHD proc type list
380  grmhd_init_mpi_processtypes(*MPI_COMM_LOCAL_GRMHD,processtypelistlocal_grmhd,sizeproclistlocal_grmhd);
381 
382 
383 
384 #endif
385 
386 
387 }
388 
389 
390 
392 void grray_init_mpi_liaisonmode(
393  int **processtypelistlocal_world,int *sizeproclistlocal_world,
394  int **processtypelistlocal_grmhd_liaison,int *sizeproclistlocal_grmhd_liaison,
395  int **processtypelistlocal_grray_liaison,int *sizeproclistlocal_grray_liaison,
396  int **processtypelistlocal_grray,int *sizeproclistlocal_grray,
397  int **processtypelistlocal_liaison_from_grray,int *sizeproclistlocal_liaison_from_grray,
398  MPI_Group *MPI_GROUP_LOCAL_WORLD,
399  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
400  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
401  MPI_Group *MPI_GROUP_LOCAL_GRRAY, MPI_Comm *MPI_COMM_LOCAL_GRRAY,
402  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRRAY)
403 {
404  void init_MPI_group_grray_liaison_split(
405  int *processtypelistlocal_grray_liaison,int sizeproclistlocal_grray_liaison,
406  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
407  MPI_Group *MPI_GROUP_LOCAL_GRRAY, MPI_Comm *MPI_COMM_LOCAL_GRRAY,
408  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRRAY);
409 
410 #if(USEMPILIAISON)
411 
412 
413  // get processtypes
414  grray_init_mpi_processtypes(MPI_COMM_WORLD,processtypelistlocal_world,sizeproclistlocal_world);
415 
416 
417  // global group sets:
418  init_MPI_group_grmhd_grray_liaison(
419  *processtypelistlocal_world,*sizeproclistlocal_world,
420  MPI_GROUP_LOCAL_WORLD,
421  MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_COMM_LOCAL_GRMHD_LIAISON,
422  MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_COMM_LOCAL_GRRAY_LIAISON);
423 
424  // ensure really NULL
425  if(*MPI_COMM_LOCAL_GRMHD_LIAISON!=MPI_COMM_NULL){
426  stderrfprintf("GRMHD_LIAISON still has members of GRRAY!\n"); fflush(stderr);
427  exit(1);
428  }
429 
430  // get processtypes for newly created subgroups
431  // 1) Get GRRAY+LIAISON proc type list
432  grray_init_mpi_processtypes(*MPI_COMM_LOCAL_GRRAY_LIAISON,processtypelistlocal_grray_liaison,sizeproclistlocal_grray_liaison);
433 
434  // liaison split from liaison+grray group
435  init_MPI_group_grray_liaison_split(
436  *processtypelistlocal_grray_liaison,*sizeproclistlocal_grray_liaison,
437  MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_COMM_LOCAL_GRRAY_LIAISON,
438  MPI_GROUP_LOCAL_GRRAY, MPI_COMM_LOCAL_GRRAY,
439  MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_COMM_LOCAL_LIAISON_FROM_GRRAY);
440 
441  // ensure really NULL
442  if(*MPI_COMM_LOCAL_LIAISON_FROM_GRRAY!=MPI_COMM_NULL){
443  stderrfprintf("MPI_COMM_LOCAL_LIAISON_FROM_GRRAY still has members of GRRAY!\n"); fflush(stderr);
444  exit(1);
445  }
446 
447  // get processtypes for newly created subgroups
448  // 1) Get GRRAY proc type list
449  grray_init_mpi_processtypes(*MPI_COMM_LOCAL_GRRAY,processtypelistlocal_grray,sizeproclistlocal_grray);
450 
451 
452 
453 #endif
454 
455 
456 }
457 
458 
459 
460 
461 
462 
463 
464 
465 
466 
471 void init_MPI_group_grmhd_grray_liaison(
472  int *processtypelistlocal,int sizeproclistlocal,
473  MPI_Group *MPI_GROUP_LOCAL_WORLD,
474  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
475  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON)
476 {
477  int *ranks;
478  int i,j,k,numranks;
479  int procsdone;
480  int sizetemp;
481 
482  // allocate things that are truenumprocs in size
483  ranks=(int*)malloc(sizeof(int)*truenumprocs);
484  if(ranks==NULL){
485  stderrfprintf("Problem allocating memory for ranks with truenumprocs=%d\n",truenumprocs); fflush(stderr);
486  myexit(3876252356);
487  }
488  for(i=0;i<truenumprocs;i++) ranks[i]=0;
489 
490 
491 #if(USEMPILIAISON)
492 
494  // get group for MPI_COMM_WORLD
495  MPI_Comm_group(MPI_COMM_WORLD, MPI_GROUP_LOCAL_WORLD);
496 
497 
498  // stderrfprintf("sizeproclistlocal=%d\n",sizeproclistlocal);
499  // for(i=0;i<sizeproclistlocal;i++){
500  // stderrfprintf("it[%d]=%d\n",i,processtypelistlocal[i]); fflush(stderr);
501  // }
502 
503 
505  // create GRMHD+LIAISON GROUP ensuring GRMHD's are ordered as originally (in case ordered at startup) and LIAISON's at end
506  j=0;
507  for(i=0;i<sizeproclistlocal;i++){
508  if(processtypelistlocal[i]==BINARYGRMHDTYPE){
509  ranks[j]=i;
510  j++;
511  }
512  }
513  for(i=0;i<sizeproclistlocal;i++){
514  if(processtypelistlocal[i]==BINARYLIAISONTYPE){
515  ranks[j]=i;
516  j++;
517  }
518  }
519  numranks=j;
520 
521 
522  // now create group and communicator
523  // Below "ranks" is more primitive than GRMHD code ranks, so below shouldn't be mapped
524  MPI_Group_incl(*MPI_GROUP_LOCAL_WORLD, numranks, ranks, MPI_GROUP_LOCAL_GRMHD_LIAISON);
525  MPI_Comm_create(MPI_COMM_WORLD, *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_COMM_LOCAL_GRMHD_LIAISON);
526 
527 
528 
530  // create GRRAY+LIAISON GROUP ensuring GRRAY's are ordered as originally (in case ordered at startup) and LIAISON's at end
531  j=0;
532  for(i=0;i<sizeproclistlocal;i++){
533  if(processtypelistlocal[i]==BINARYGRRAYTYPE){
534  ranks[j]=i;
535  j++;
536  }
537  }
538  for(i=0;i<sizeproclistlocal;i++){
539  if(processtypelistlocal[i]==BINARYLIAISONTYPE){
540  ranks[j]=i;
541  j++;
542  }
543  }
544  numranks=j;
545 
546  // now create group and communicator
547  MPI_Group_incl(*MPI_GROUP_LOCAL_WORLD, numranks, ranks, MPI_GROUP_LOCAL_GRRAY_LIAISON);
548  MPI_Comm_create(MPI_COMM_WORLD, *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_COMM_LOCAL_GRRAY_LIAISON);
549 
550  // stderrfprintf("numranks=%d\n",numranks);
551  // for(i=0;i<numranks;i++){
552  // stderrfprintf("ranks[%d]=%d\n",i,ranks[i]); fflush(stderr);
553  // }
554 
555 
556 #endif // end if USEMPILIAISON
557 
558  free(ranks);
559 
560 }
561 
562 
563 
564 
565 
571 void init_MPI_group_grmhd_grray_liaison_split(
572  int *processtypelistlocal_grmhd_liaison,int sizeproclistlocal_grmhd_liaison,
573  int *processtypelistlocal_grray_liaison,int sizeproclistlocal_grray_liaison,
574  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
575  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
576  MPI_Group *MPI_GROUP_LOCAL_GRMHD, MPI_Comm *MPI_COMM_LOCAL_GRMHD,
577  MPI_Group *MPI_GROUP_LOCAL_GRRAY, MPI_Comm *MPI_COMM_LOCAL_GRRAY,
578  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRMHD,
579  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRRAY)
580 {
581 
582  void init_MPI_group_grmhd_liaison_split(
583  int *processtypelistlocal_grmhd_liaison,int sizeproclistlocal_grmhd_liaison,
584  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
585  MPI_Group *MPI_GROUP_LOCAL_GRMHD, MPI_Comm *MPI_COMM_LOCAL_GRMHD,
586  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRMHD);
587  void init_MPI_group_grray_liaison_split(
588  int *processtypelistlocal_grray_liaison,int sizeproclistlocal_grray_liaison,
589  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
590  MPI_Group *MPI_GROUP_LOCAL_GRRAY, MPI_Comm *MPI_COMM_LOCAL_GRRAY,
591  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRRAY);
592 
593 
594 #if(USEMPILIAISON)
595 
596  // split GRMHD+LIAISON (uses same function used by GRMHD code)
597  init_MPI_group_grmhd_liaison_split(processtypelistlocal_grmhd_liaison,sizeproclistlocal_grmhd_liaison,
598  MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_COMM_LOCAL_GRMHD_LIAISON,
599  MPI_GROUP_LOCAL_GRMHD, MPI_COMM_LOCAL_GRMHD,
600  MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_COMM_LOCAL_LIAISON_FROM_GRMHD);
601 
602 
603  // split GRRAY+LIAISON (uses same function used by GRRAY code)
604  init_MPI_group_grray_liaison_split(processtypelistlocal_grray_liaison,sizeproclistlocal_grray_liaison,
605  MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_COMM_LOCAL_GRRAY_LIAISON,
606  MPI_GROUP_LOCAL_GRRAY, MPI_COMM_LOCAL_GRRAY,
607  MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_COMM_LOCAL_LIAISON_FROM_GRRAY);
608 
609 
610 
611 #endif
612 
613 
614 }
615 
616 
617 
620 void init_MPI_group_grmhd_liaison_split(
621  int *processtypelistlocal,int sizeproclistlocal,
622  MPI_Group *MPI_GROUP_LOCAL_GRMHD_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRMHD_LIAISON,
623  MPI_Group *MPI_GROUP_LOCAL_GRMHD, MPI_Comm *MPI_COMM_LOCAL_GRMHD,
624  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRMHD)
625 {
626  int *ranks;
627  int i,j,k,numranks;
628  int procsdone;
629 
630 
631  // allocate things that are truenumprocs in size
632  ranks=(int*)malloc(sizeof(int)*truenumprocs);
633  if(ranks==NULL){
634  stderrfprintf("Problem allocating memory for ranks with truenumprocs=%d\n",truenumprocs); fflush(stderr);
635  myexit(3876252356);
636  }
637  for(i=0;i<truenumprocs;i++) ranks[i]=0;
638 
639 
640 #if(USEMPILIAISON)
641 
642 
644  // decompose GRMHD+LIAISON GROUP
645  for(i=0,j=0;i<sizeproclistlocal;i++){
646  if(processtypelistlocal[i]==BINARYGRMHDTYPE){
647  ranks[j]=i;
648  j++;
649  }
650  }
651  numranks=j;
652 
653  // now create group and communicator
654  MPI_Group_incl(*MPI_GROUP_LOCAL_GRMHD_LIAISON, numranks, ranks, MPI_GROUP_LOCAL_GRMHD);
655  MPI_Comm_create(*MPI_COMM_LOCAL_GRMHD_LIAISON, *MPI_GROUP_LOCAL_GRMHD, MPI_COMM_LOCAL_GRMHD);
656 
657 
658 
660  // decompose GRRAY+LIAISON GROUP
661  for(i=0,j=0;i<sizeproclistlocal;i++){
662  if(processtypelistlocal[i]==BINARYLIAISONTYPE){
663  ranks[j]=i;
664  j++;
665  }
666  }
667  numranks=j;
668 
669  // now create group and communicator
670  MPI_Group_incl(*MPI_GROUP_LOCAL_GRMHD_LIAISON, numranks, ranks, MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD);
671  MPI_Comm_create(*MPI_COMM_LOCAL_GRMHD_LIAISON, *MPI_GROUP_LOCAL_LIAISON_FROM_GRMHD, MPI_COMM_LOCAL_LIAISON_FROM_GRMHD);
672 
673 
674 #endif
675 
676  free(ranks);
677 
678 }
679 
680 
681 
684 void init_MPI_group_grray_liaison_split(
685  int *processtypelistlocal,int sizeproclistlocal,
686  MPI_Group *MPI_GROUP_LOCAL_GRRAY_LIAISON, MPI_Comm *MPI_COMM_LOCAL_GRRAY_LIAISON,
687  MPI_Group *MPI_GROUP_LOCAL_GRRAY, MPI_Comm *MPI_COMM_LOCAL_GRRAY,
688  MPI_Group *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_Comm *MPI_COMM_LOCAL_LIAISON_FROM_GRRAY)
689 {
690  int *ranks;
691  int i,j,k,numranks;
692  int procsdone;
693 
694 
695  // allocate things that are truenumprocs in size
696  ranks=(int*)malloc(sizeof(int)*truenumprocs);
697  if(ranks==NULL){
698  stderrfprintf("Problem allocating memory for ranks with truenumprocs=%d\n",truenumprocs); fflush(stderr);
699  myexit(3876252356);
700  }
701  for(i=0;i<truenumprocs;i++) ranks[i]=0;
702 
703 
704 
705 #if(USEMPILIAISON)
706 
707 
709  // decompose GRRAY+LIAISON GROUP
710  for(i=0,j=0;i<sizeproclistlocal;i++){
711  if(processtypelistlocal[i]==BINARYGRRAYTYPE){
712  ranks[j]=i;
713  j++;
714  }
715  }
716  numranks=j;
717 
718  // now create group and communicator
719  MPI_Group_incl(*MPI_GROUP_LOCAL_GRRAY_LIAISON, numranks, ranks, MPI_GROUP_LOCAL_GRRAY);
720  MPI_Comm_create(*MPI_COMM_LOCAL_GRRAY_LIAISON, *MPI_GROUP_LOCAL_GRRAY, MPI_COMM_LOCAL_GRRAY);
721 
722 
723 
725  // decompose GRRAY+LIAISON GROUP
726  for(i=0,j=0;i<sizeproclistlocal;i++){
727  if(processtypelistlocal[i]==BINARYLIAISONTYPE){
728  ranks[j]=i;
729  j++;
730  }
731  }
732  numranks=j;
733 
734  // now create group and communicator
735  MPI_Group_incl(*MPI_GROUP_LOCAL_GRRAY_LIAISON, numranks, ranks, MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY);
736  MPI_Comm_create(*MPI_COMM_LOCAL_GRRAY_LIAISON, *MPI_GROUP_LOCAL_LIAISON_FROM_GRRAY, MPI_COMM_LOCAL_LIAISON_FROM_GRRAY);
737 
738 #endif
739 
740 
741  free(ranks);
742 
743 }
744 
745 
746 // when using these communicators, must make sure the call to a communication using it isn't done by a non-member cpu!
747 // (above: stupid, I know, should just skip if non-member cpu tries a function)
748 
749 //void init_MPI_group_liaison_free(void)
750 //{
751 // MPI_Comm_free(&combound[i]); // messy since makes nonmember have NULL comm. Should make as is, and if not member, then skip
752 // MPI_Group_free(&grprem[i]);
753 //}
754 
755 
756 
757 
758 
759 
761 int final_myexit(void)
762 {
763 #if(USEMPI)
764  // must abort since no clear to communicate to other cpus
765  MPI_Abort(MPI_COMM_WORLD, 1);
766 #endif
767 
768  stderrfprintf( "END\n");
769  fflush(stderr);
770  exit(0);
771 
772  return(0);
773 }
774 
775 
776 
777 
778 
779 
780 
781 
782 
783 
784 
785 
786 
787 
788 
789 #else
790 
791 //
792 // NOT normal liaison mode (for normal GRMHD or GRRAY run or debugging)
793 //
795 
796 
797 
798 
799 
802 int final_myexit(void)
803 {
804 #if(USEMPI)
805  // finish up MPI
806  // Barrier required
807  MPI_Barrier(MPI_COMM_WORLD);
808  MPI_Finalize();
809 #endif
810 
811  stderrfprintf( "END\n");
812  fflush(stderr);
813  exit(0);
814 
815 
816  return(0);
817 }
818 
819 
820 
821 
822 
823 
824 
825 #if(DOINGLIAISONTYPECODE)
826 
827 
829 void liaison_init_mpi_liaisonmode_globalset(void)
830 {
831 
832 
833 #if(USEMPILIAISON)
834 
836  // get group for MPI_COMM_WORLD
837  MPI_Comm_group(MPI_COMM_WORLD, &MPI_GROUP_WORLD);
838  MPI_COMM_LIAISON_FROM_GRMHD=MPI_COMM_WORLD;
839  MPI_GROUP_LIAISON_FROM_GRMHD=MPI_GROUP_WORLD;
840 
841  liaison_init_mpi_processtypes(MPI_COMM_WORLD, &processtypelist_world, &sizeproclist_world);
842  liaison_init_mpi_processtypes(MPI_COMM_LIAISON_FROM_GRMHD, &processtypelist_liaison_from_grmhd, &sizeproclist_liaison_from_grmhd);
843 
844 #else
845  sizeproclist_world=sizeproclist_liaison_from_grmhd=1;
846  processtypelist_world=(int*)malloc(sizeof(int)*1);
847  processtypelist_liaison_from_grmhd=(int*)malloc(sizeof(int)*1);
848  if(processtypelist_world==NULL || processtypelist_liaison_from_grmhd==NULL){
849  stderrfprintf("Couldn't allocate memory for processtypelist_world,liaison_from_grmhd\n");
850  exit(1);
851  }
852  processtypelist_world[0]=BINARYLIAISONTYPE;
853  processtypelist_liaison_from_grmhd[0]=BINARYLIAISONTYPE;
854 #endif
855 
856  // needed for parts of liaison code that uses set-up of grid in grmhd code
857  sizeproclist_grmhd=sizeproclist_world;
858 
859 #if(USEMPILIAISON||USEMPIGRMHD)
860  MPI_GROUP_GRMHD=MPI_GROUP_LIAISON_FROM_GRMHD;
861  MPI_COMM_GRMHD=MPI_COMM_LIAISON_FROM_GRMHD;
862 #endif
863 
864 
865 }
866 
867 
868 
869 
870 #endif // end if liaison type code
871 
872 
873 
874 #if(DOINGGRMHDTYPECODE)
875 
878 void grmhd_init_mpi_liaisonmode_globalset(void)
879 {
880 
881 
882 #if(USEMPIGRMHD)
883 
885  // get group for MPI_COMM_WORLD
886  stderrfprintf("MPICOMM1\n");
887  MPI_Comm_group(MPI_COMM_WORLD, &MPI_GROUP_WORLD);
888  stderrfprintf("MPICOMM2\n");
889  MPI_COMM_GRMHD=MPI_COMM_WORLD;
890  stderrfprintf("MPICOMM3\n");
891  MPI_GROUP_GRMHD=MPI_GROUP_WORLD;
892 
893  stderrfprintf("MPICOMM4\n");
894  grmhd_init_mpi_processtypes(MPI_COMM_WORLD, &processtypelist_world, &sizeproclist_world);
895  stderrfprintf("MPICOMM5\n");
896  grmhd_init_mpi_processtypes(MPI_COMM_GRMHD, &processtypelist_grmhd, &sizeproclist_grmhd);
897  stderrfprintf("MPICOMM6\n");
898 
899 #else
900  sizeproclist_world=sizeproclist_grmhd=1;
901  processtypelist_world=(int*)malloc(sizeof(int)*1);
902  processtypelist_grmhd=(int*)malloc(sizeof(int)*1);
903  if(processtypelist_world==NULL || processtypelist_grmhd==NULL){
904  stderrfprintf("Couldn't allocate memory for processtypelist_world,grmhd\n");
905  exit(1);
906  }
907  processtypelist_world[0]=BINARYGRMHDTYPE;
908  processtypelist_grmhd[0]=BINARYGRMHDTYPE;
909 #endif
910 
911 }
912 
913 
914 #endif // end if grmhd type code
915 
916 
917 
918 #if(DOINGGRRAYTYPECODE)
919 
922 void grray_init_mpi_liaisonmode_globalset(void)
923 {
924 
925 
926 #if(USEMPIGRRAY)
927 
929  // get group for MPI_COMM_WORLD
930  MPI_Comm_group(MPI_COMM_WORLD, &MPI_GROUP_WORLD);
931  MPI_COMM_GRRAY=MPI_COMM_WORLD;
932  MPI_GROUP_GRRAY=MPI_GROUP_WORLD;
933 
934  grray_init_mpi_processtypes(MPI_COMM_WORLD, &processtypelist_world, &sizeproclist_world);
935  grray_init_mpi_processtypes(MPI_COMM_GRRAY, &processtypelist_grray, &sizeproclist_grray);
936 
937 #else
938  sizeproclist_world=sizeproclist_grray=1;
939  processtypelist_world=(int*)malloc(sizeof(int)*1);
940  processtypelist_grray=(int*)malloc(sizeof(int)*1);
941  if(processtypelist_world==NULL || processtypelist_grray==NULL){
942  stderrfprintf("Couldn't allocate memory for processtypelist_world,grray\n");
943  exit(1);
944  }
945  processtypelist_world[0]=BINARYGRRAYTYPE;
946  processtypelist_grray[0]=BINARYGRRAYTYPE;
947 #endif
948 
949 }
950 
951 
952 
953 #endif // end if grray type code
954 
955 
956 #endif // end if not doing any liaison mode
957 
958 
959 
Generated on Fri May 20 2016 15:52:34 for HARM by   doxygen 1.8.3.1