Actual source code: sf.c
petsc-3.7.5 2017-01-01
1: #include <petsc/private/sfimpl.h> /*I "petscsf.h" I*/
2: #include <petscctable.h>
4: /* Logging support */
5: PetscLogEvent PETSCSF_SetGraph, PETSCSF_BcastBegin, PETSCSF_BcastEnd, PETSCSF_ReduceBegin, PETSCSF_ReduceEnd, PETSCSF_FetchAndOpBegin, PETSCSF_FetchAndOpEnd;
7: #if defined(PETSC_USE_DEBUG)
8: # define PetscSFCheckGraphSet(sf,arg) do { \
9: if (PetscUnlikely(!(sf)->graphset)) \
10: SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Must call PetscSFSetGraph() on argument %D \"%s\" before %s()",(arg),#sf,PETSC_FUNCTION_NAME); \
11: } while (0)
12: #else
13: # define PetscSFCheckGraphSet(sf,arg) do {} while (0)
14: #endif
16: const char *const PetscSFDuplicateOptions[] = {"CONFONLY","RANKS","GRAPH","PetscSFDuplicateOption","PETSCSF_DUPLICATE_",0};
20: /*@C
21: PetscSFCreate - create a star forest communication context
23: Not Collective
25: Input Arguments:
26: . comm - communicator on which the star forest will operate
28: Output Arguments:
29: . sf - new star forest context
31: Level: intermediate
33: .seealso: PetscSFSetGraph(), PetscSFDestroy()
34: @*/
35: PetscErrorCode PetscSFCreate(MPI_Comm comm,PetscSF *sf)
36: {
38: PetscSF b;
42: PetscSFInitializePackage();
44: PetscHeaderCreate(b,PETSCSF_CLASSID,"PetscSF","Star Forest","PetscSF",comm,PetscSFDestroy,PetscSFView);
46: b->nroots = -1;
47: b->nleaves = -1;
48: b->nranks = -1;
49: b->rankorder = PETSC_TRUE;
50: b->ingroup = MPI_GROUP_NULL;
51: b->outgroup = MPI_GROUP_NULL;
52: b->graphset = PETSC_FALSE;
54: *sf = b;
55: return(0);
56: }
60: /*@C
61: PetscSFReset - Reset a star forest so that different sizes or neighbors can be used
63: Collective
65: Input Arguments:
66: . sf - star forest
68: Level: advanced
70: .seealso: PetscSFCreate(), PetscSFSetGraph(), PetscSFDestroy()
71: @*/
72: PetscErrorCode PetscSFReset(PetscSF sf)
73: {
78: sf->mine = NULL;
79: PetscFree(sf->mine_alloc);
80: sf->remote = NULL;
81: PetscFree(sf->remote_alloc);
82: PetscFree4(sf->ranks,sf->roffset,sf->rmine,sf->rremote);
83: PetscFree(sf->degree);
84: if (sf->ingroup != MPI_GROUP_NULL) {MPI_Group_free(&sf->ingroup);}
85: if (sf->outgroup != MPI_GROUP_NULL) {MPI_Group_free(&sf->outgroup);}
86: PetscSFDestroy(&sf->multi);
87: sf->graphset = PETSC_FALSE;
88: if (sf->ops->Reset) {(*sf->ops->Reset)(sf);}
89: sf->setupcalled = PETSC_FALSE;
90: return(0);
91: }
95: /*@C
96: PetscSFSetType - set the PetscSF communication implementation
98: Collective on PetscSF
100: Input Parameters:
101: + sf - the PetscSF context
102: - type - a known method
104: Options Database Key:
105: . -sf_type <type> - Sets the method; use -help for a list
106: of available methods (for instance, window, pt2pt, neighbor)
108: Notes:
109: See "include/petscsf.h" for available methods (for instance)
110: + PETSCSFWINDOW - MPI-2/3 one-sided
111: - PETSCSFBASIC - basic implementation using MPI-1 two-sided
113: Level: intermediate
115: .keywords: PetscSF, set, type
117: .seealso: PetscSFType, PetscSFCreate()
118: @*/
119: PetscErrorCode PetscSFSetType(PetscSF sf,PetscSFType type)
120: {
121: PetscErrorCode ierr,(*r)(PetscSF);
122: PetscBool match;
128: PetscObjectTypeCompare((PetscObject)sf,type,&match);
129: if (match) return(0);
131: PetscFunctionListFind(PetscSFList,type,&r);
132: if (!r) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_UNKNOWN_TYPE,"Unable to find requested PetscSF type %s",type);
133: /* Destroy the previous private PetscSF context */
134: if (sf->ops->Destroy) {
135: (*(sf)->ops->Destroy)(sf);
136: }
137: PetscMemzero(sf->ops,sizeof(*sf->ops));
138: PetscObjectChangeTypeName((PetscObject)sf,type);
139: (*r)(sf);
140: return(0);
141: }
145: /*@
146: PetscSFDestroy - destroy star forest
148: Collective
150: Input Arguments:
151: . sf - address of star forest
153: Level: intermediate
155: .seealso: PetscSFCreate(), PetscSFReset()
156: @*/
157: PetscErrorCode PetscSFDestroy(PetscSF *sf)
158: {
162: if (!*sf) return(0);
164: if (--((PetscObject)(*sf))->refct > 0) {*sf = 0; return(0);}
165: PetscSFReset(*sf);
166: if ((*sf)->ops->Destroy) {(*(*sf)->ops->Destroy)(*sf);}
167: PetscHeaderDestroy(sf);
168: return(0);
169: }
173: /*@
174: PetscSFSetUp - set up communication structures
176: Collective
178: Input Arguments:
179: . sf - star forest communication object
181: Level: beginner
183: .seealso: PetscSFSetFromOptions(), PetscSFSetType()
184: @*/
185: PetscErrorCode PetscSFSetUp(PetscSF sf)
186: {
190: if (sf->setupcalled) return(0);
191: if (!((PetscObject)sf)->type_name) {PetscSFSetType(sf,PETSCSFBASIC);}
192: if (sf->ops->SetUp) {(*sf->ops->SetUp)(sf);}
193: sf->setupcalled = PETSC_TRUE;
194: return(0);
195: }
199: /*@C
200: PetscSFSetFromOptions - set PetscSF options using the options database
202: Logically Collective
204: Input Arguments:
205: . sf - star forest
207: Options Database Keys:
208: + -sf_type - implementation type, see PetscSFSetType()
209: - -sf_rank_order - sort composite points for gathers and scatters in rank order, gathers are non-deterministic otherwise
211: Level: intermediate
213: .keywords: KSP, set, from, options, database
215: .seealso: PetscSFWindowSetSyncType()
216: @*/
217: PetscErrorCode PetscSFSetFromOptions(PetscSF sf)
218: {
219: PetscSFType deft;
220: char type[256];
222: PetscBool flg;
226: PetscObjectOptionsBegin((PetscObject)sf);
227: deft = ((PetscObject)sf)->type_name ? ((PetscObject)sf)->type_name : PETSCSFBASIC;
228: PetscOptionsFList("-sf_type","PetscSF implementation type","PetscSFSetType",PetscSFList,deft,type,256,&flg);
229: PetscSFSetType(sf,flg ? type : deft);
230: PetscOptionsBool("-sf_rank_order","sort composite points for gathers and scatters in rank order, gathers are non-deterministic otherwise","PetscSFSetRankOrder",sf->rankorder,&sf->rankorder,NULL);
231: if (sf->ops->SetFromOptions) {(*sf->ops->SetFromOptions)(PetscOptionsObject,sf);}
232: PetscOptionsEnd();
233: return(0);
234: }
238: /*@C
239: PetscSFSetRankOrder - sort multi-points for gathers and scatters by rank order
241: Logically Collective
243: Input Arguments:
244: + sf - star forest
245: - flg - PETSC_TRUE to sort, PETSC_FALSE to skip sorting (lower setup cost, but non-deterministic)
247: Level: advanced
249: .seealso: PetscSFGatherBegin(), PetscSFScatterBegin()
250: @*/
251: PetscErrorCode PetscSFSetRankOrder(PetscSF sf,PetscBool flg)
252: {
257: if (sf->multi) SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_WRONGSTATE,"Rank ordering must be set before first call to PetscSFGatherBegin() or PetscSFScatterBegin()");
258: sf->rankorder = flg;
259: return(0);
260: }
264: /*@C
265: PetscSFSetGraph - Set a parallel star forest
267: Collective
269: Input Arguments:
270: + sf - star forest
271: . nroots - number of root vertices on the current process (these are possible targets for other process to attach leaves)
272: . nleaves - number of leaf vertices on the current process, each of these references a root on any process
273: . ilocal - locations of leaves in leafdata buffers, pass NULL for contiguous storage
274: . localmode - copy mode for ilocal
275: . iremote - remote locations of root vertices for each leaf on the current process
276: - remotemode - copy mode for iremote
278: Level: intermediate
280: .seealso: PetscSFCreate(), PetscSFView(), PetscSFGetGraph()
281: @*/
282: PetscErrorCode PetscSFSetGraph(PetscSF sf,PetscInt nroots,PetscInt nleaves,const PetscInt *ilocal,PetscCopyMode localmode,const PetscSFNode *iremote,PetscCopyMode remotemode)
283: {
284: PetscErrorCode ierr;
285: PetscTable table;
286: PetscTablePosition pos;
287: PetscMPIInt size;
288: PetscInt i,*rcount,*ranks;
292: PetscLogEventBegin(PETSCSF_SetGraph,sf,0,0,0);
295: if (nroots < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"roots %D, cannot be negative",nroots);
296: if (nleaves < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"nleaves %D, cannot be negative",nleaves);
297: PetscSFReset(sf);
298: sf->nroots = nroots;
299: sf->nleaves = nleaves;
300: if (ilocal) {
301: switch (localmode) {
302: case PETSC_COPY_VALUES:
303: PetscMalloc1(nleaves,&sf->mine_alloc);
304: sf->mine = sf->mine_alloc;
305: PetscMemcpy(sf->mine,ilocal,nleaves*sizeof(*sf->mine));
306: sf->minleaf = PETSC_MAX_INT;
307: sf->maxleaf = PETSC_MIN_INT;
308: for (i=0; i<nleaves; i++) {
309: sf->minleaf = PetscMin(sf->minleaf,ilocal[i]);
310: sf->maxleaf = PetscMax(sf->maxleaf,ilocal[i]);
311: }
312: break;
313: case PETSC_OWN_POINTER:
314: sf->mine_alloc = (PetscInt*)ilocal;
315: sf->mine = sf->mine_alloc;
316: break;
317: case PETSC_USE_POINTER:
318: sf->mine = (PetscInt*)ilocal;
319: break;
320: default: SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_OUTOFRANGE,"Unknown localmode");
321: }
322: }
323: if (!ilocal || nleaves > 0) {
324: sf->minleaf = 0;
325: sf->maxleaf = nleaves - 1;
326: }
327: switch (remotemode) {
328: case PETSC_COPY_VALUES:
329: PetscMalloc1(nleaves,&sf->remote_alloc);
330: sf->remote = sf->remote_alloc;
331: PetscMemcpy(sf->remote,iremote,nleaves*sizeof(*sf->remote));
332: break;
333: case PETSC_OWN_POINTER:
334: sf->remote_alloc = (PetscSFNode*)iremote;
335: sf->remote = sf->remote_alloc;
336: break;
337: case PETSC_USE_POINTER:
338: sf->remote = (PetscSFNode*)iremote;
339: break;
340: default: SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_OUTOFRANGE,"Unknown remotemode");
341: }
343: MPI_Comm_size(PetscObjectComm((PetscObject)sf),&size);
344: PetscTableCreate(10,size,&table);
345: for (i=0; i<nleaves; i++) {
346: /* Log 1-based rank */
347: PetscTableAdd(table,iremote[i].rank+1,1,ADD_VALUES);
348: }
349: PetscTableGetCount(table,&sf->nranks);
350: PetscMalloc4(sf->nranks,&sf->ranks,sf->nranks+1,&sf->roffset,nleaves,&sf->rmine,nleaves,&sf->rremote);
351: PetscMalloc2(sf->nranks,&rcount,sf->nranks,&ranks);
352: PetscTableGetHeadPosition(table,&pos);
353: for (i=0; i<sf->nranks; i++) {
354: PetscTableGetNext(table,&pos,&ranks[i],&rcount[i]);
355: ranks[i]--; /* Convert back to 0-based */
356: }
357: PetscTableDestroy(&table);
358: PetscSortIntWithArray(sf->nranks,ranks,rcount);
359: sf->roffset[0] = 0;
360: for (i=0; i<sf->nranks; i++) {
361: PetscMPIIntCast(ranks[i],sf->ranks+i);
362: sf->roffset[i+1] = sf->roffset[i] + rcount[i];
363: rcount[i] = 0;
364: }
365: for (i=0; i<nleaves; i++) {
366: PetscInt lo,hi,irank;
367: /* Search for index of iremote[i].rank in sf->ranks */
368: lo = 0; hi = sf->nranks;
369: while (hi - lo > 1) {
370: PetscInt mid = lo + (hi - lo)/2;
371: if (iremote[i].rank < sf->ranks[mid]) hi = mid;
372: else lo = mid;
373: }
374: if (hi - lo == 1 && iremote[i].rank == sf->ranks[lo]) irank = lo;
375: else SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Could not find rank %D in array",iremote[i].rank);
376: sf->rmine[sf->roffset[irank] + rcount[irank]] = ilocal ? ilocal[i] : i;
377: sf->rremote[sf->roffset[irank] + rcount[irank]] = iremote[i].index;
378: rcount[irank]++;
379: }
380: PetscFree2(rcount,ranks);
381: #if !defined(PETSC_USE_64BIT_INDICES)
382: if (nroots == PETSC_DETERMINE) {
383: /* Jed, if you have a better way to do this, put it in */
384: PetscInt *numRankLeaves, *leafOff, *leafIndices, *numRankRoots, *rootOff, *rootIndices, maxRoots = 0;
386: /* All to all to determine number of leaf indices from each (you can do this using Scan and asynch messages) */
387: PetscMalloc4(size,&numRankLeaves,size+1,&leafOff,size,&numRankRoots,size+1,&rootOff);
388: PetscMemzero(numRankLeaves, size * sizeof(PetscInt));
389: for (i = 0; i < nleaves; ++i) ++numRankLeaves[iremote[i].rank];
390: MPI_Alltoall(numRankLeaves, 1, MPIU_INT, numRankRoots, 1, MPIU_INT, PetscObjectComm((PetscObject)sf));
391: /* Could set nroots to this maximum */
392: for (i = 0; i < size; ++i) maxRoots += numRankRoots[i];
394: /* Gather all indices */
395: PetscMalloc2(nleaves,&leafIndices,maxRoots,&rootIndices);
396: leafOff[0] = 0;
397: for (i = 0; i < size; ++i) leafOff[i+1] = leafOff[i] + numRankLeaves[i];
398: for (i = 0; i < nleaves; ++i) leafIndices[leafOff[iremote[i].rank]++] = iremote[i].index;
399: leafOff[0] = 0;
400: for (i = 0; i < size; ++i) leafOff[i+1] = leafOff[i] + numRankLeaves[i];
401: rootOff[0] = 0;
402: for (i = 0; i < size; ++i) rootOff[i+1] = rootOff[i] + numRankRoots[i];
403: MPI_Alltoallv(leafIndices, numRankLeaves, leafOff, MPIU_INT, rootIndices, numRankRoots, rootOff, MPIU_INT, PetscObjectComm((PetscObject)sf));
404: /* Sort and reduce */
405: PetscSortRemoveDupsInt(&maxRoots, rootIndices);
406: PetscFree2(leafIndices,rootIndices);
407: PetscFree4(numRankLeaves,leafOff,numRankRoots,rootOff);
408: sf->nroots = maxRoots;
409: }
410: #endif
412: sf->graphset = PETSC_TRUE;
413: PetscLogEventEnd(PETSCSF_SetGraph,sf,0,0,0);
414: return(0);
415: }
419: /*@C
420: PetscSFCreateInverseSF - given a PetscSF in which all vertices have degree 1, creates the inverse map
422: Collective
424: Input Arguments:
425: . sf - star forest to invert
427: Output Arguments:
428: . isf - inverse of sf
430: Level: advanced
432: Notes:
433: All roots must have degree 1.
435: The local space may be a permutation, but cannot be sparse.
437: .seealso: PetscSFSetGraph()
438: @*/
439: PetscErrorCode PetscSFCreateInverseSF(PetscSF sf,PetscSF *isf)
440: {
442: PetscMPIInt rank;
443: PetscInt i,nroots,nleaves,maxlocal,count,*newilocal;
444: const PetscInt *ilocal;
445: PetscSFNode *roots,*leaves;
448: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
449: PetscSFGetGraph(sf,&nroots,&nleaves,&ilocal,NULL);
450: for (i=0,maxlocal=0; i<nleaves; i++) maxlocal = PetscMax(maxlocal,(ilocal ? ilocal[i] : i)+1);
451: PetscMalloc2(nroots,&roots,maxlocal,&leaves);
452: for (i=0; i<maxlocal; i++) {
453: leaves[i].rank = rank;
454: leaves[i].index = i;
455: }
456: for (i=0; i <nroots; i++) {
457: roots[i].rank = -1;
458: roots[i].index = -1;
459: }
460: PetscSFReduceBegin(sf,MPIU_2INT,leaves,roots,MPIU_REPLACE);
461: PetscSFReduceEnd(sf,MPIU_2INT,leaves,roots,MPIU_REPLACE);
463: /* Check whether our leaves are sparse */
464: for (i=0,count=0; i<nroots; i++) if (roots[i].rank >= 0) count++;
465: if (count == nroots) newilocal = NULL;
466: else { /* Index for sparse leaves and compact "roots" array (which is to become our leaves). */
467: PetscMalloc1(count,&newilocal);
468: for (i=0,count=0; i<nroots; i++) {
469: if (roots[i].rank >= 0) {
470: newilocal[count] = i;
471: roots[count].rank = roots[i].rank;
472: roots[count].index = roots[i].index;
473: count++;
474: }
475: }
476: }
478: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_CONFONLY,isf);
479: PetscSFSetGraph(*isf,maxlocal,count,newilocal,PETSC_OWN_POINTER,roots,PETSC_COPY_VALUES);
480: PetscFree2(roots,leaves);
481: return(0);
482: }
486: /*@
487: PetscSFDuplicate - duplicate a PetscSF, optionally preserving rank connectivity and graph
489: Collective
491: Input Arguments:
492: + sf - communication object to duplicate
493: - opt - PETSCSF_DUPLICATE_CONFONLY, PETSCSF_DUPLICATE_RANKS, or PETSCSF_DUPLICATE_GRAPH (see PetscSFDuplicateOption)
495: Output Arguments:
496: . newsf - new communication object
498: Level: beginner
500: .seealso: PetscSFCreate(), PetscSFSetType(), PetscSFSetGraph()
501: @*/
502: PetscErrorCode PetscSFDuplicate(PetscSF sf,PetscSFDuplicateOption opt,PetscSF *newsf)
503: {
507: PetscSFCreate(PetscObjectComm((PetscObject)sf),newsf);
508: PetscSFSetType(*newsf,((PetscObject)sf)->type_name);
509: if (sf->ops->Duplicate) {(*sf->ops->Duplicate)(sf,opt,*newsf);}
510: if (opt == PETSCSF_DUPLICATE_GRAPH) {
511: PetscInt nroots,nleaves;
512: const PetscInt *ilocal;
513: const PetscSFNode *iremote;
514: PetscSFGetGraph(sf,&nroots,&nleaves,&ilocal,&iremote);
515: PetscSFSetGraph(*newsf,nroots,nleaves,ilocal,PETSC_COPY_VALUES,iremote,PETSC_COPY_VALUES);
516: }
517: return(0);
518: }
522: /*@C
523: PetscSFGetGraph - Get the graph specifying a parallel star forest
525: Not Collective
527: Input Arguments:
528: . sf - star forest
530: Output Arguments:
531: + nroots - number of root vertices on the current process (these are possible targets for other process to attach leaves)
532: . nleaves - number of leaf vertices on the current process, each of these references a root on any process
533: . ilocal - locations of leaves in leafdata buffers
534: - iremote - remote locations of root vertices for each leaf on the current process
536: Level: intermediate
538: .seealso: PetscSFCreate(), PetscSFView(), PetscSFSetGraph()
539: @*/
540: PetscErrorCode PetscSFGetGraph(PetscSF sf,PetscInt *nroots,PetscInt *nleaves,const PetscInt **ilocal,const PetscSFNode **iremote)
541: {
545: /* We are not currently requiring that the graph is set, thus returning nroots=-1 if it has not been set */
546: /* if (!sf->graphset) SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_WRONGSTATE,"Graph has not been set, must call PetscSFSetGraph()"); */
547: if (nroots) *nroots = sf->nroots;
548: if (nleaves) *nleaves = sf->nleaves;
549: if (ilocal) *ilocal = sf->mine;
550: if (iremote) *iremote = sf->remote;
551: return(0);
552: }
556: /*@C
557: PetscSFGetLeafRange - Get the active leaf ranges
559: Not Collective
561: Input Arguments:
562: . sf - star forest
564: Output Arguments:
565: + minleaf - minimum active leaf on this process
566: - maxleaf - maximum active leaf on this process
568: Level: developer
570: .seealso: PetscSFCreate(), PetscSFView(), PetscSFSetGraph(), PetscSFGetGraph()
571: @*/
572: PetscErrorCode PetscSFGetLeafRange(PetscSF sf,PetscInt *minleaf,PetscInt *maxleaf)
573: {
577: if (minleaf) *minleaf = sf->minleaf;
578: if (maxleaf) *maxleaf = sf->maxleaf;
579: return(0);
580: }
584: /*@C
585: PetscSFView - view a star forest
587: Collective
589: Input Arguments:
590: + sf - star forest
591: - viewer - viewer to display graph, for example PETSC_VIEWER_STDOUT_WORLD
593: Level: beginner
595: .seealso: PetscSFCreate(), PetscSFSetGraph()
596: @*/
597: PetscErrorCode PetscSFView(PetscSF sf,PetscViewer viewer)
598: {
599: PetscErrorCode ierr;
600: PetscBool iascii;
601: PetscViewerFormat format;
605: if (!viewer) {PetscViewerASCIIGetStdout(PetscObjectComm((PetscObject)sf),&viewer);}
608: PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
609: if (iascii) {
610: PetscMPIInt rank;
611: PetscInt i,j;
613: PetscObjectPrintClassNamePrefixType((PetscObject)sf,viewer);
614: PetscViewerASCIIPushTab(viewer);
615: if (sf->ops->View) {(*sf->ops->View)(sf,viewer);}
616: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
617: PetscViewerASCIIPushSynchronized(viewer);
618: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of roots=%D, leaves=%D, remote ranks=%D\n",rank,sf->nroots,sf->nleaves,sf->nranks);
619: for (i=0; i<sf->nleaves; i++) {
620: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %D <- (%D,%D)\n",rank,sf->mine ? sf->mine[i] : i,sf->remote[i].rank,sf->remote[i].index);
621: }
622: PetscViewerFlush(viewer);
623: PetscViewerGetFormat(viewer,&format);
624: if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
625: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Roots referenced by my leaves, by rank\n",rank);
626: for (i=0; i<sf->nranks; i++) {
627: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %d: %D edges\n",rank,sf->ranks[i],sf->roffset[i+1]-sf->roffset[i]);
628: for (j=sf->roffset[i]; j<sf->roffset[i+1]; j++) {
629: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %D <- %D\n",rank,sf->rmine[j],sf->rremote[j]);
630: }
631: }
632: }
633: PetscViewerFlush(viewer);
634: PetscViewerASCIIPopSynchronized(viewer);
635: PetscViewerASCIIPopTab(viewer);
636: }
637: return(0);
638: }
642: /*@C
643: PetscSFGetRanks - Get ranks and number of vertices referenced by leaves on this process
645: Not Collective
647: Input Arguments:
648: . sf - star forest
650: Output Arguments:
651: + nranks - number of ranks referenced by local part
652: . ranks - array of ranks
653: . roffset - offset in rmine/rremote for each rank (length nranks+1)
654: . rmine - concatenated array holding local indices referencing each remote rank
655: - rremote - concatenated array holding remote indices referenced for each remote rank
657: Level: developer
659: .seealso: PetscSFSetGraph()
660: @*/
661: PetscErrorCode PetscSFGetRanks(PetscSF sf,PetscInt *nranks,const PetscMPIInt **ranks,const PetscInt **roffset,const PetscInt **rmine,const PetscInt **rremote)
662: {
666: if (nranks) *nranks = sf->nranks;
667: if (ranks) *ranks = sf->ranks;
668: if (roffset) *roffset = sf->roffset;
669: if (rmine) *rmine = sf->rmine;
670: if (rremote) *rremote = sf->rremote;
671: return(0);
672: }
676: /*@C
677: PetscSFGetGroups - gets incoming and outgoing process groups
679: Collective
681: Input Argument:
682: . sf - star forest
684: Output Arguments:
685: + incoming - group of origin processes for incoming edges (leaves that reference my roots)
686: - outgoing - group of destination processes for outgoing edges (roots that I reference)
688: Level: developer
690: .seealso: PetscSFGetWindow(), PetscSFRestoreWindow()
691: @*/
692: PetscErrorCode PetscSFGetGroups(PetscSF sf,MPI_Group *incoming,MPI_Group *outgoing)
693: {
695: MPI_Group group;
698: if (sf->ingroup == MPI_GROUP_NULL) {
699: PetscInt i;
700: const PetscInt *indegree;
701: PetscMPIInt rank,*outranks,*inranks;
702: PetscSFNode *remote;
703: PetscSF bgcount;
705: /* Compute the number of incoming ranks */
706: PetscMalloc1(sf->nranks,&remote);
707: for (i=0; i<sf->nranks; i++) {
708: remote[i].rank = sf->ranks[i];
709: remote[i].index = 0;
710: }
711: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_CONFONLY,&bgcount);
712: PetscSFSetGraph(bgcount,1,sf->nranks,NULL,PETSC_COPY_VALUES,remote,PETSC_OWN_POINTER);
713: PetscSFComputeDegreeBegin(bgcount,&indegree);
714: PetscSFComputeDegreeEnd(bgcount,&indegree);
716: /* Enumerate the incoming ranks */
717: PetscMalloc2(indegree[0],&inranks,sf->nranks,&outranks);
718: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
719: for (i=0; i<sf->nranks; i++) outranks[i] = rank;
720: PetscSFGatherBegin(bgcount,MPI_INT,outranks,inranks);
721: PetscSFGatherEnd(bgcount,MPI_INT,outranks,inranks);
722: MPI_Comm_group(PetscObjectComm((PetscObject)sf),&group);
723: MPI_Group_incl(group,indegree[0],inranks,&sf->ingroup);
724: MPI_Group_free(&group);
725: PetscFree2(inranks,outranks);
726: PetscSFDestroy(&bgcount);
727: }
728: *incoming = sf->ingroup;
730: if (sf->outgroup == MPI_GROUP_NULL) {
731: MPI_Comm_group(PetscObjectComm((PetscObject)sf),&group);
732: MPI_Group_incl(group,sf->nranks,sf->ranks,&sf->outgroup);
733: MPI_Group_free(&group);
734: }
735: *outgoing = sf->outgroup;
736: return(0);
737: }
741: /*@C
742: PetscSFGetMultiSF - gets the inner SF implemeting gathers and scatters
744: Collective
746: Input Argument:
747: . sf - star forest that may contain roots with 0 or with more than 1 vertex
749: Output Arguments:
750: . multi - star forest with split roots, such that each root has degree exactly 1
752: Level: developer
754: Notes:
756: In most cases, users should use PetscSFGatherBegin() and PetscSFScatterBegin() instead of manipulating multi
757: directly. Since multi satisfies the stronger condition that each entry in the global space has exactly one incoming
758: edge, it is a candidate for future optimization that might involve its removal.
760: .seealso: PetscSFSetGraph(), PetscSFGatherBegin(), PetscSFScatterBegin()
761: @*/
762: PetscErrorCode PetscSFGetMultiSF(PetscSF sf,PetscSF *multi)
763: {
769: if (sf->nroots < 0) { /* Graph has not been set yet; why do we need this? */
770: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);
771: *multi = sf->multi;
772: return(0);
773: }
774: if (!sf->multi) {
775: const PetscInt *indegree;
776: PetscInt i,*inoffset,*outones,*outoffset,maxlocal;
777: PetscSFNode *remote;
778: PetscSFComputeDegreeBegin(sf,&indegree);
779: PetscSFComputeDegreeEnd(sf,&indegree);
780: for (i=0,maxlocal=0; i<sf->nleaves; i++) maxlocal = PetscMax(maxlocal,(sf->mine ? sf->mine[i] : i)+1);
781: PetscMalloc3(sf->nroots+1,&inoffset,maxlocal,&outones,maxlocal,&outoffset);
782: inoffset[0] = 0;
783: for (i=0; i<sf->nroots; i++) inoffset[i+1] = inoffset[i] + indegree[i];
784: for (i=0; i<maxlocal; i++) outones[i] = 1;
785: PetscSFFetchAndOpBegin(sf,MPIU_INT,inoffset,outones,outoffset,MPI_SUM);
786: PetscSFFetchAndOpEnd(sf,MPIU_INT,inoffset,outones,outoffset,MPI_SUM);
787: for (i=0; i<sf->nroots; i++) inoffset[i] -= indegree[i]; /* Undo the increment */
788: #if 0
789: #if defined(PETSC_USE_DEBUG) /* Check that the expected number of increments occurred */
790: for (i=0; i<sf->nroots; i++) {
791: if (inoffset[i] + indegree[i] != inoffset[i+1]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Incorrect result after PetscSFFetchAndOp");
792: }
793: #endif
794: #endif
795: PetscMalloc1(sf->nleaves,&remote);
796: for (i=0; i<sf->nleaves; i++) {
797: remote[i].rank = sf->remote[i].rank;
798: remote[i].index = outoffset[sf->mine ? sf->mine[i] : i];
799: }
800: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);
801: PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,sf->mine,PETSC_COPY_VALUES,remote,PETSC_OWN_POINTER);
802: if (sf->rankorder) { /* Sort the ranks */
803: PetscMPIInt rank;
804: PetscInt *inranks,*newoffset,*outranks,*newoutoffset,*tmpoffset,maxdegree;
805: PetscSFNode *newremote;
806: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
807: for (i=0,maxdegree=0; i<sf->nroots; i++) maxdegree = PetscMax(maxdegree,indegree[i]);
808: PetscMalloc5(sf->multi->nroots,&inranks,sf->multi->nroots,&newoffset,maxlocal,&outranks,maxlocal,&newoutoffset,maxdegree,&tmpoffset);
809: for (i=0; i<maxlocal; i++) outranks[i] = rank;
810: PetscSFReduceBegin(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);
811: PetscSFReduceEnd(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);
812: /* Sort the incoming ranks at each vertex, build the inverse map */
813: for (i=0; i<sf->nroots; i++) {
814: PetscInt j;
815: for (j=0; j<indegree[i]; j++) tmpoffset[j] = j;
816: PetscSortIntWithArray(indegree[i],inranks+inoffset[i],tmpoffset);
817: for (j=0; j<indegree[i]; j++) newoffset[inoffset[i] + tmpoffset[j]] = inoffset[i] + j;
818: }
819: PetscSFBcastBegin(sf->multi,MPIU_INT,newoffset,newoutoffset);
820: PetscSFBcastEnd(sf->multi,MPIU_INT,newoffset,newoutoffset);
821: PetscMalloc1(sf->nleaves,&newremote);
822: for (i=0; i<sf->nleaves; i++) {
823: newremote[i].rank = sf->remote[i].rank;
824: newremote[i].index = newoutoffset[sf->mine ? sf->mine[i] : i];
825: }
826: PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,sf->mine,PETSC_COPY_VALUES,newremote,PETSC_OWN_POINTER);
827: PetscFree5(inranks,newoffset,outranks,newoutoffset,tmpoffset);
828: }
829: PetscFree3(inoffset,outones,outoffset);
830: }
831: *multi = sf->multi;
832: return(0);
833: }
837: /*@C
838: PetscSFCreateEmbeddedSF - removes edges from all but the selected roots, does not remap indices
840: Collective
842: Input Arguments:
843: + sf - original star forest
844: . nroots - number of roots to select on this process
845: - selected - selected roots on this process
847: Output Arguments:
848: . newsf - new star forest
850: Level: advanced
852: Note:
853: To use the new PetscSF, it may be necessary to know the indices of the leaves that are still participating. This can
854: be done by calling PetscSFGetGraph().
856: .seealso: PetscSFSetGraph(), PetscSFGetGraph()
857: @*/
858: PetscErrorCode PetscSFCreateEmbeddedSF(PetscSF sf,PetscInt nroots,const PetscInt *selected,PetscSF *newsf)
859: {
860: PetscInt *rootdata, *leafdata, *ilocal;
861: PetscSFNode *iremote;
862: PetscInt leafsize = 0, nleaves = 0, n, i;
869: if (sf->mine) for (i = 0; i < sf->nleaves; ++i) {leafsize = PetscMax(leafsize, sf->mine[i]+1);}
870: else leafsize = sf->nleaves;
871: PetscCalloc2(sf->nroots,&rootdata,leafsize,&leafdata);
872: for (i=0; i<nroots; ++i) rootdata[selected[i]] = 1;
873: PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);
874: PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);
876: for (i = 0; i < leafsize; ++i) nleaves += leafdata[i];
877: PetscMalloc1(nleaves,&ilocal);
878: PetscMalloc1(nleaves,&iremote);
879: for (i = 0, n = 0; i < sf->nleaves; ++i) {
880: const PetscInt lidx = sf->mine ? sf->mine[i] : i;
882: if (leafdata[lidx]) {
883: ilocal[n] = lidx;
884: iremote[n].rank = sf->remote[i].rank;
885: iremote[n].index = sf->remote[i].index;
886: ++n;
887: }
888: }
889: if (n != nleaves) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "There is a size mismatch in the SF embedding, %d != %d", n, nleaves);
890: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,newsf);
891: PetscSFSetGraph(*newsf,sf->nroots,nleaves,ilocal,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);
892: PetscFree2(rootdata,leafdata);
893: return(0);
894: }
898: /*@C
899: PetscSFCreateEmbeddedLeafSF - removes edges from all but the selected leaves, does not remap indices
901: Collective
903: Input Arguments:
904: + sf - original star forest
905: . nleaves - number of leaves to select on this process
906: - selected - selected leaves on this process
908: Output Arguments:
909: . newsf - new star forest
911: Level: advanced
913: .seealso: PetscSFCreateEmbeddedSF(), PetscSFSetGraph(), PetscSFGetGraph()
914: @*/
915: PetscErrorCode PetscSFCreateEmbeddedLeafSF(PetscSF sf, PetscInt nleaves, const PetscInt *selected, PetscSF *newsf)
916: {
917: PetscSFNode *iremote;
918: PetscInt *ilocal;
919: PetscInt i;
926: PetscMalloc1(nleaves, &ilocal);
927: PetscMalloc1(nleaves, &iremote);
928: for (i = 0; i < nleaves; ++i) {
929: const PetscInt l = selected[i];
931: ilocal[i] = sf->mine ? sf->mine[l] : l;
932: iremote[i].rank = sf->remote[l].rank;
933: iremote[i].index = sf->remote[l].index;
934: }
935: PetscSFDuplicate(sf, PETSCSF_DUPLICATE_RANKS, newsf);
936: PetscSFSetGraph(*newsf, sf->nroots, nleaves, ilocal, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER);
937: return(0);
938: }
942: /*@C
943: PetscSFBcastBegin - begin pointwise broadcast to be concluded with call to PetscSFBcastEnd()
945: Collective on PetscSF
947: Input Arguments:
948: + sf - star forest on which to communicate
949: . unit - data type associated with each node
950: - rootdata - buffer to broadcast
952: Output Arguments:
953: . leafdata - buffer to update with values from each leaf's respective root
955: Level: intermediate
957: .seealso: PetscSFCreate(), PetscSFSetGraph(), PetscSFView(), PetscSFBcastEnd(), PetscSFReduceBegin()
958: @*/
959: PetscErrorCode PetscSFBcastBegin(PetscSF sf,MPI_Datatype unit,const void *rootdata,void *leafdata)
960: {
965: PetscSFCheckGraphSet(sf,1);
966: PetscLogEventBegin(PETSCSF_BcastBegin,sf,0,0,0);
967: PetscSFSetUp(sf);
968: (*sf->ops->BcastBegin)(sf,unit,rootdata,leafdata);
969: PetscLogEventEnd(PETSCSF_BcastBegin,sf,0,0,0);
970: return(0);
971: }
975: /*@C
976: PetscSFBcastEnd - end a broadcast operation started with PetscSFBcastBegin()
978: Collective
980: Input Arguments:
981: + sf - star forest
982: . unit - data type
983: - rootdata - buffer to broadcast
985: Output Arguments:
986: . leafdata - buffer to update with values from each leaf's respective root
988: Level: intermediate
990: .seealso: PetscSFSetGraph(), PetscSFReduceEnd()
991: @*/
992: PetscErrorCode PetscSFBcastEnd(PetscSF sf,MPI_Datatype unit,const void *rootdata,void *leafdata)
993: {
998: PetscSFCheckGraphSet(sf,1);
999: PetscLogEventBegin(PETSCSF_BcastEnd,sf,0,0,0);
1000: PetscSFSetUp(sf);
1001: (*sf->ops->BcastEnd)(sf,unit,rootdata,leafdata);
1002: PetscLogEventEnd(PETSCSF_BcastEnd,sf,0,0,0);
1003: return(0);
1004: }
1008: /*@C
1009: PetscSFReduceBegin - begin reduction of leafdata into rootdata, to be completed with call to PetscSFReduceEnd()
1011: Collective
1013: Input Arguments:
1014: + sf - star forest
1015: . unit - data type
1016: . leafdata - values to reduce
1017: - op - reduction operation
1019: Output Arguments:
1020: . rootdata - result of reduction of values from all leaves of each root
1022: Level: intermediate
1024: .seealso: PetscSFBcastBegin()
1025: @*/
1026: PetscErrorCode PetscSFReduceBegin(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *rootdata,MPI_Op op)
1027: {
1032: PetscSFCheckGraphSet(sf,1);
1033: PetscLogEventBegin(PETSCSF_ReduceBegin,sf,0,0,0);
1034: PetscSFSetUp(sf);
1035: (sf->ops->ReduceBegin)(sf,unit,leafdata,rootdata,op);
1036: PetscLogEventEnd(PETSCSF_ReduceBegin,sf,0,0,0);
1037: return(0);
1038: }
1042: /*@C
1043: PetscSFReduceEnd - end a reduction operation started with PetscSFReduceBegin()
1045: Collective
1047: Input Arguments:
1048: + sf - star forest
1049: . unit - data type
1050: . leafdata - values to reduce
1051: - op - reduction operation
1053: Output Arguments:
1054: . rootdata - result of reduction of values from all leaves of each root
1056: Level: intermediate
1058: .seealso: PetscSFSetGraph(), PetscSFBcastEnd()
1059: @*/
1060: PetscErrorCode PetscSFReduceEnd(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *rootdata,MPI_Op op)
1061: {
1066: PetscSFCheckGraphSet(sf,1);
1067: PetscLogEventBegin(PETSCSF_ReduceEnd,sf,0,0,0);
1068: PetscSFSetUp(sf);
1069: (*sf->ops->ReduceEnd)(sf,unit,leafdata,rootdata,op);
1070: PetscLogEventEnd(PETSCSF_ReduceEnd,sf,0,0,0);
1071: return(0);
1072: }
1076: /*@C
1077: PetscSFComputeDegreeBegin - begin computation of degree for each root vertex, to be completed with PetscSFComputeDegreeEnd()
1079: Collective
1081: Input Arguments:
1082: . sf - star forest
1084: Output Arguments:
1085: . degree - degree of each root vertex
1087: Level: advanced
1089: .seealso: PetscSFGatherBegin()
1090: @*/
1091: PetscErrorCode PetscSFComputeDegreeBegin(PetscSF sf,const PetscInt **degree)
1092: {
1097: PetscSFCheckGraphSet(sf,1);
1099: if (!sf->degreeknown) {
1100: PetscInt i,maxlocal;
1101: if (sf->degree) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Calls to PetscSFComputeDegreeBegin() cannot be nested.");
1102: for (i=0,maxlocal=0; i<sf->nleaves; i++) maxlocal = PetscMax(maxlocal,(sf->mine ? sf->mine[i] : i)+1);
1103: PetscMalloc1(sf->nroots,&sf->degree);
1104: PetscMalloc1(maxlocal,&sf->degreetmp);
1105: for (i=0; i<sf->nroots; i++) sf->degree[i] = 0;
1106: for (i=0; i<maxlocal; i++) sf->degreetmp[i] = 1;
1107: PetscSFReduceBegin(sf,MPIU_INT,sf->degreetmp,sf->degree,MPI_SUM);
1108: }
1109: *degree = NULL;
1110: return(0);
1111: }
1115: /*@C
1116: PetscSFComputeDegreeEnd - complete computation of degree for each root vertex, started with PetscSFComputeDegreeBegin()
1118: Collective
1120: Input Arguments:
1121: . sf - star forest
1123: Output Arguments:
1124: . degree - degree of each root vertex
1126: Level: developer
1128: .seealso:
1129: @*/
1130: PetscErrorCode PetscSFComputeDegreeEnd(PetscSF sf,const PetscInt **degree)
1131: {
1136: PetscSFCheckGraphSet(sf,1);
1137: if (!sf->degreeknown) {
1138: PetscSFReduceEnd(sf,MPIU_INT,sf->degreetmp,sf->degree,MPI_SUM);
1139: PetscFree(sf->degreetmp);
1141: sf->degreeknown = PETSC_TRUE;
1142: }
1143: *degree = sf->degree;
1144: return(0);
1145: }
1149: /*@C
1150: PetscSFFetchAndOpBegin - begin operation that fetches values from root and updates atomically by applying operation using my leaf value, to be completed with PetscSFFetchAndOpEnd()
1152: Collective
1154: Input Arguments:
1155: + sf - star forest
1156: . unit - data type
1157: . leafdata - leaf values to use in reduction
1158: - op - operation to use for reduction
1160: Output Arguments:
1161: + rootdata - root values to be updated, input state is seen by first process to perform an update
1162: - leafupdate - state at each leaf's respective root immediately prior to my atomic update
1164: Level: advanced
1166: Note:
1167: The update is only atomic at the granularity provided by the hardware. Different roots referenced by the same process
1168: might be updated in a different order. Furthermore, if a composite type is used for the unit datatype, atomicity is
1169: not guaranteed across the whole vertex. Therefore, this function is mostly only used with primitive types such as
1170: integers.
1172: .seealso: PetscSFComputeDegreeBegin(), PetscSFReduceBegin(), PetscSFSetGraph()
1173: @*/
1174: PetscErrorCode PetscSFFetchAndOpBegin(PetscSF sf,MPI_Datatype unit,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op op)
1175: {
1180: PetscSFCheckGraphSet(sf,1);
1181: PetscLogEventBegin(PETSCSF_FetchAndOpBegin,sf,0,0,0);
1182: PetscSFSetUp(sf);
1183: (*sf->ops->FetchAndOpBegin)(sf,unit,rootdata,leafdata,leafupdate,op);
1184: PetscLogEventEnd(PETSCSF_FetchAndOpBegin,sf,0,0,0);
1185: return(0);
1186: }
1190: /*@C
1191: PetscSFFetchAndOpEnd - end operation started in matching call to PetscSFFetchAndOpBegin() to fetch values from roots and update atomically by applying operation using my leaf value
1193: Collective
1195: Input Arguments:
1196: + sf - star forest
1197: . unit - data type
1198: . leafdata - leaf values to use in reduction
1199: - op - operation to use for reduction
1201: Output Arguments:
1202: + rootdata - root values to be updated, input state is seen by first process to perform an update
1203: - leafupdate - state at each leaf's respective root immediately prior to my atomic update
1205: Level: advanced
1207: .seealso: PetscSFComputeDegreeEnd(), PetscSFReduceEnd(), PetscSFSetGraph()
1208: @*/
1209: PetscErrorCode PetscSFFetchAndOpEnd(PetscSF sf,MPI_Datatype unit,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op op)
1210: {
1215: PetscSFCheckGraphSet(sf,1);
1216: PetscLogEventBegin(PETSCSF_FetchAndOpEnd,sf,0,0,0);
1217: PetscSFSetUp(sf);
1218: (*sf->ops->FetchAndOpEnd)(sf,unit,rootdata,leafdata,leafupdate,op);
1219: PetscLogEventEnd(PETSCSF_FetchAndOpEnd,sf,0,0,0);
1220: return(0);
1221: }
1225: /*@C
1226: PetscSFGatherBegin - begin pointwise gather of all leaves into multi-roots, to be completed with PetscSFGatherEnd()
1228: Collective
1230: Input Arguments:
1231: + sf - star forest
1232: . unit - data type
1233: - leafdata - leaf data to gather to roots
1235: Output Argument:
1236: . multirootdata - root buffer to gather into, amount of space per root is equal to its degree
1238: Level: intermediate
1240: .seealso: PetscSFComputeDegreeBegin(), PetscSFScatterBegin()
1241: @*/
1242: PetscErrorCode PetscSFGatherBegin(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *multirootdata)
1243: {
1245: PetscSF multi;
1249: PetscSFGetMultiSF(sf,&multi);
1250: PetscSFReduceBegin(multi,unit,leafdata,multirootdata,MPIU_REPLACE);
1251: return(0);
1252: }
1256: /*@C
1257: PetscSFGatherEnd - ends pointwise gather operation that was started with PetscSFGatherBegin()
1259: Collective
1261: Input Arguments:
1262: + sf - star forest
1263: . unit - data type
1264: - leafdata - leaf data to gather to roots
1266: Output Argument:
1267: . multirootdata - root buffer to gather into, amount of space per root is equal to its degree
1269: Level: intermediate
1271: .seealso: PetscSFComputeDegreeEnd(), PetscSFScatterEnd()
1272: @*/
1273: PetscErrorCode PetscSFGatherEnd(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *multirootdata)
1274: {
1276: PetscSF multi;
1280: PetscSFCheckGraphSet(sf,1);
1281: PetscSFSetUp(sf);
1282: PetscSFGetMultiSF(sf,&multi);
1283: PetscSFReduceEnd(multi,unit,leafdata,multirootdata,MPIU_REPLACE);
1284: return(0);
1285: }
1289: /*@C
1290: PetscSFScatterBegin - begin pointwise scatter operation from multi-roots to leaves, to be completed with PetscSFScatterEnd()
1292: Collective
1294: Input Arguments:
1295: + sf - star forest
1296: . unit - data type
1297: - multirootdata - root buffer to send to each leaf, one unit of data per leaf
1299: Output Argument:
1300: . leafdata - leaf data to be update with personal data from each respective root
1302: Level: intermediate
1304: .seealso: PetscSFComputeDegreeBegin(), PetscSFScatterBegin()
1305: @*/
1306: PetscErrorCode PetscSFScatterBegin(PetscSF sf,MPI_Datatype unit,const void *multirootdata,void *leafdata)
1307: {
1309: PetscSF multi;
1313: PetscSFCheckGraphSet(sf,1);
1314: PetscSFSetUp(sf);
1315: PetscSFGetMultiSF(sf,&multi);
1316: PetscSFBcastBegin(multi,unit,multirootdata,leafdata);
1317: return(0);
1318: }
1322: /*@C
1323: PetscSFScatterEnd - ends pointwise scatter operation that was started with PetscSFScatterBegin()
1325: Collective
1327: Input Arguments:
1328: + sf - star forest
1329: . unit - data type
1330: - multirootdata - root buffer to send to each leaf, one unit of data per leaf
1332: Output Argument:
1333: . leafdata - leaf data to be update with personal data from each respective root
1335: Level: intermediate
1337: .seealso: PetscSFComputeDegreeEnd(), PetscSFScatterEnd()
1338: @*/
1339: PetscErrorCode PetscSFScatterEnd(PetscSF sf,MPI_Datatype unit,const void *multirootdata,void *leafdata)
1340: {
1342: PetscSF multi;
1346: PetscSFCheckGraphSet(sf,1);
1347: PetscSFSetUp(sf);
1348: PetscSFGetMultiSF(sf,&multi);
1349: PetscSFBcastEnd(multi,unit,multirootdata,leafdata);
1350: return(0);
1351: }
1355: /*@
1356: PetscSFCompose - Compose a new PetscSF equivalent to action to PetscSFs
1358: Input Parameters:
1359: + sfA - The first PetscSF
1360: - sfB - The second PetscSF
1362: Output Parameters:
1363: . sfBA - equvalent PetscSF for applying A then B
1365: Level: developer
1367: .seealso: PetscSF, PetscSFGetGraph(), PetscSFSetGraph()
1368: @*/
1369: PetscErrorCode PetscSFCompose(PetscSF sfA, PetscSF sfB, PetscSF *sfBA)
1370: {
1371: MPI_Comm comm;
1372: const PetscSFNode *remotePointsA, *remotePointsB;
1373: PetscSFNode *remotePointsBA;
1374: const PetscInt *localPointsA, *localPointsB;
1375: PetscInt numRootsA, numLeavesA, numRootsB, numLeavesB;
1376: PetscErrorCode ierr;
1381: PetscObjectGetComm((PetscObject) sfA, &comm);
1382: PetscSFGetGraph(sfA, &numRootsA, &numLeavesA, &localPointsA, &remotePointsA);
1383: PetscSFGetGraph(sfB, &numRootsB, &numLeavesB, &localPointsB, &remotePointsB);
1384: PetscMalloc1(numLeavesB, &remotePointsBA);
1385: PetscSFBcastBegin(sfB, MPIU_2INT, remotePointsA, remotePointsBA);
1386: PetscSFBcastEnd(sfB, MPIU_2INT, remotePointsA, remotePointsBA);
1387: PetscSFCreate(comm, sfBA);
1388: PetscSFSetGraph(*sfBA, numRootsA, numLeavesB, localPointsB, PETSC_COPY_VALUES, remotePointsBA, PETSC_OWN_POINTER);
1389: return(0);
1390: }