Actual source code: redundant.c
1: #define PETSCKSP_DLL
3: /*
4: This file defines a "solve the problem redundantly on each subgroup of processor" preconditioner.
5: */
6: #include private/pcimpl.h
7: #include petscksp.h
9: typedef struct {
10: KSP ksp;
11: PC pc; /* actual preconditioner used on each processor */
12: Vec xsub,ysub; /* vectors of a subcommunicator to hold parallel vectors of pc->comm */
13: Vec xdup,ydup; /* parallel vector that congregates xsub or ysub facilitating vector scattering */
14: Mat pmats; /* matrix and optional preconditioner matrix belong to a subcommunicator */
15: VecScatter scatterin,scatterout; /* scatter used to move all values to each processor group (subcommunicator) */
16: PetscTruth useparallelmat;
17: PetscSubcomm psubcomm;
18: PetscInt nsubcomm; /* num of data structure PetscSubcomm */
19: } PC_Redundant;
23: static PetscErrorCode PCView_Redundant(PC pc,PetscViewer viewer)
24: {
25: PC_Redundant *red = (PC_Redundant*)pc->data;
27: PetscMPIInt rank;
28: PetscTruth iascii,isstring;
29: PetscViewer sviewer,subviewer;
30: PetscInt color = red->psubcomm->color;
33: MPI_Comm_rank(pc->comm,&rank);
34: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
35: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
36: if (iascii) {
37: PetscViewerASCIIPrintf(viewer," Redundant preconditioner: First (color=0) of %D PCs follows\n",red->nsubcomm);
38: PetscViewerGetSubcomm(viewer,red->pc->comm,&subviewer);
39: if (!color) { /* only view first redundant pc */
40: PetscViewerASCIIPushTab(viewer);
41: KSPView(red->ksp,subviewer);
42: PetscViewerASCIIPopTab(viewer);
43: }
44: PetscViewerRestoreSubcomm(viewer,red->pc->comm,&subviewer);
45: } else if (isstring) { /* not test it yet! */
46: PetscViewerStringSPrintf(viewer," Redundant solver preconditioner");
47: PetscViewerGetSingleton(viewer,&sviewer);
48: if (!rank) {
49: KSPView(red->ksp,sviewer);
50: }
51: PetscViewerRestoreSingleton(viewer,&sviewer);
52: } else {
53: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for PC redundant",((PetscObject)viewer)->type_name);
54: }
55: return(0);
56: }
58: #include include/private/matimpl.h
61: static PetscErrorCode PCSetUp_Redundant(PC pc)
62: {
63: PC_Redundant *red = (PC_Redundant*)pc->data;
65: PetscInt mstart,mend,mlocal,m,mlocal_sub,rstart_sub,rend_sub,mloc_sub;
66: PetscMPIInt size;
67: MatReuse reuse = MAT_INITIAL_MATRIX;
68: MatStructure str = DIFFERENT_NONZERO_PATTERN;
69: MPI_Comm comm = pc->comm,subcomm;
70: Vec vec;
71: PetscMPIInt subsize,subrank;
72: const char *prefix;
73: KSP subksp;
76: MatGetVecs(pc->pmat,&vec,0);
77: VecGetSize(vec,&m);
79: if (!pc->setupcalled) {
80: PetscSubcommCreate(comm,red->nsubcomm,&red->psubcomm);
81: PetscLogObjectMemory(pc,sizeof(PetscSubcomm));
83: /* create a new PC that processors in each subcomm have copy of */
84: subcomm = red->psubcomm->comm;
85: KSPCreate(subcomm,&subksp);
86: PetscLogObjectParent(pc,subksp);
87: KSPSetType(subksp,KSPPREONLY);
88: KSPGetPC(subksp,&red->pc);
89: PCSetType(red->pc,PCLU);
91: PCGetOptionsPrefix(pc,&prefix);
92: KSPSetOptionsPrefix(subksp,prefix);
93: KSPAppendOptionsPrefix(subksp,"redundant_");
94: red->ksp = subksp;
96: /* create working vectors xsub/ysub and xdup/ydup */
97: VecGetLocalSize(vec,&mlocal);
98: VecGetOwnershipRange(vec,&mstart,&mend);
100: /* get local size of xsub/ysub */
101: MPI_Comm_size(subcomm,&subsize);
102: MPI_Comm_rank(subcomm,&subrank);
103: rstart_sub = pc->pmat->rmap.range[red->psubcomm->n*subrank]; /* rstart in xsub/ysub */
104: if (subrank+1 < subsize){
105: rend_sub = pc->pmat->rmap.range[red->psubcomm->n*(subrank+1)];
106: } else {
107: rend_sub = m;
108: }
109: mloc_sub = rend_sub - rstart_sub;
110: VecCreateMPI(subcomm,mloc_sub,PETSC_DECIDE,&red->ysub);
111: /* create xsub with empty local arrays, because xdup's arrays will be placed into it */
112: VecCreateMPIWithArray(subcomm,mloc_sub,PETSC_DECIDE,PETSC_NULL,&red->xsub);
114: /* create xdup and ydup. ydup has empty local arrays because ysub's arrays will be place into it.
115: Note: we use communicator dupcomm, not pc->comm! */
116: VecCreateMPI(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,&red->xdup);
117: VecCreateMPIWithArray(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,PETSC_NULL,&red->ydup);
118:
119: /* create vec scatters */
120: if (!red->scatterin){
121: IS is1,is2;
122: PetscInt *idx1,*idx2,i,j,k;
124: PetscMalloc(2*red->psubcomm->n*mlocal*sizeof(PetscInt),&idx1);
125: idx2 = idx1 + red->psubcomm->n*mlocal;
126: j = 0;
127: for (k=0; k<red->psubcomm->n; k++){
128: for (i=mstart; i<mend; i++){
129: idx1[j] = i;
130: idx2[j++] = i + m*k;
131: }
132: }
133: ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,&is1);
134: ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,&is2);
135: VecScatterCreate(vec,is1,red->xdup,is2,&red->scatterin);
136: ISDestroy(is1);
137: ISDestroy(is2);
139: ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*m,1,&is1);
140: ISCreateStride(comm,mlocal,mstart,1,&is2);
141: VecScatterCreate(red->xdup,is1,vec,is2,&red->scatterout);
142: ISDestroy(is1);
143: ISDestroy(is2);
144: PetscFree(idx1);
145: }
146: }
147: VecDestroy(vec);
149: /* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
150: MPI_Comm_size(comm,&size);
151: if (size == 1) {
152: red->useparallelmat = PETSC_FALSE;
153: }
155: if (red->useparallelmat) {
156: if (pc->setupcalled == 1 && pc->flag == DIFFERENT_NONZERO_PATTERN) {
157: /* destroy old matrices */
158: if (red->pmats) {
159: MatDestroy(red->pmats);
160: }
161: } else if (pc->setupcalled == 1) {
162: reuse = MAT_REUSE_MATRIX;
163: str = SAME_NONZERO_PATTERN;
164: }
165:
166: /* grab the parallel matrix and put it into processors of a subcomminicator */
167: /*--------------------------------------------------------------------------*/
168: VecGetLocalSize(red->ysub,&mlocal_sub);
169: MatGetRedundantMatrix(pc->pmat,red->psubcomm->n,red->psubcomm->comm,mlocal_sub,reuse,&red->pmats);
170: /* tell PC of the subcommunicator its operators */
171: KSPSetOperators(red->ksp,red->pmats,red->pmats,str);
172: } else {
173: KSPSetOperators(red->ksp,pc->mat,pc->pmat,pc->flag);
174: }
175: if (pc->setfromoptionscalled){
176: KSPSetFromOptions(red->ksp);
177: }
178: KSPSetUp(red->ksp);
179: return(0);
180: }
184: static PetscErrorCode PCApply_Redundant(PC pc,Vec x,Vec y)
185: {
186: PC_Redundant *red = (PC_Redundant*)pc->data;
188: PetscScalar *array;
191: /* scatter x to xdup */
192: VecScatterBegin(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);
193: VecScatterEnd(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);
194:
195: /* place xdup's local array into xsub */
196: VecGetArray(red->xdup,&array);
197: VecPlaceArray(red->xsub,(const PetscScalar*)array);
199: /* apply preconditioner on each processor */
200: PCApply(red->pc,red->xsub,red->ysub);
201: VecResetArray(red->xsub);
202: VecRestoreArray(red->xdup,&array);
203:
204: /* place ysub's local array into ydup */
205: VecGetArray(red->ysub,&array);
206: VecPlaceArray(red->ydup,(const PetscScalar*)array);
208: /* scatter ydup to y */
209: VecScatterBegin(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);
210: VecScatterEnd(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);
211: VecResetArray(red->ydup);
212: VecRestoreArray(red->ysub,&array);
213: return(0);
214: }
218: static PetscErrorCode PCDestroy_Redundant(PC pc)
219: {
220: PC_Redundant *red = (PC_Redundant*)pc->data;
224: if (red->scatterin) {VecScatterDestroy(red->scatterin);}
225: if (red->scatterout) {VecScatterDestroy(red->scatterout);}
226: if (red->ysub) {VecDestroy(red->ysub);}
227: if (red->xsub) {VecDestroy(red->xsub);}
228: if (red->xdup) {VecDestroy(red->xdup);}
229: if (red->ydup) {VecDestroy(red->ydup);}
230: if (red->pmats) {
231: MatDestroy(red->pmats);
232: }
233: if (red->psubcomm) {PetscSubcommDestroy(red->psubcomm);}
234: if (red->ksp) {KSPDestroy(red->ksp);}
235: PetscFree(red);
236: return(0);
237: }
241: static PetscErrorCode PCSetFromOptions_Redundant(PC pc)
242: {
244: PC_Redundant *red = (PC_Redundant*)pc->data;
247: PetscOptionsHead("Redundant options");
248: PetscOptionsInt("-pc_redundant_number","Number of redundant pc","PCRedundantSetNumber",red->nsubcomm,&red->nsubcomm,0);
249: PetscOptionsTail();
250: return(0);
251: }
256: PetscErrorCode PCRedundantSetNumber_Redundant(PC pc,PetscInt nreds)
257: {
258: PC_Redundant *red = (PC_Redundant*)pc->data;
261: red->nsubcomm = nreds;
262: return(0);
263: }
268: /*@
269: PCRedundantSetNumber - Sets the number of redundant preconditioner contexts.
271: Collective on PC
273: Input Parameters:
274: + pc - the preconditioner context
275: - nredundant - number of redundant preconditioner contexts; for example if you are using 64 MPI processes and
276: use an nredundant of 4 there will be 4 parallel solves each on 16 = 64/4 processes.
278: Level: advanced
280: .keywords: PC, redundant solve
281: @*/
282: PetscErrorCode PCRedundantSetNumber(PC pc,PetscInt nredundant)
283: {
284: PetscErrorCode ierr,(*f)(PC,PetscInt);
288: if (nredundant <= 0) SETERRQ1(PETSC_ERR_ARG_WRONG, "num of redundant pc %D must be positive",nredundant);
289: PetscObjectQueryFunction((PetscObject)pc,"PCRedundantSetNumber_C",(void (**)(void))&f);
290: if (f) {
291: (*f)(pc,nredundant);
292: }
293: return(0);
294: }
299: PetscErrorCode PCRedundantSetScatter_Redundant(PC pc,VecScatter in,VecScatter out)
300: {
301: PC_Redundant *red = (PC_Redundant*)pc->data;
305: PetscObjectReference((PetscObject)in);
306: if (red->scatterin) { VecScatterDestroy(red->scatterin); }
307: red->scatterin = in;
308: PetscObjectReference((PetscObject)out);
309: if (red->scatterout) { VecScatterDestroy(red->scatterout); }
310: red->scatterout = out;
311: return(0);
312: }
317: /*@
318: PCRedundantSetScatter - Sets the scatter used to copy values into the
319: redundant local solve and the scatter to move them back into the global
320: vector.
322: Collective on PC
324: Input Parameters:
325: + pc - the preconditioner context
326: . in - the scatter to move the values in
327: - out - the scatter to move them out
329: Level: advanced
331: .keywords: PC, redundant solve
332: @*/
333: PetscErrorCode PCRedundantSetScatter(PC pc,VecScatter in,VecScatter out)
334: {
335: PetscErrorCode ierr,(*f)(PC,VecScatter,VecScatter);
341: PetscObjectQueryFunction((PetscObject)pc,"PCRedundantSetScatter_C",(void (**)(void))&f);
342: if (f) {
343: (*f)(pc,in,out);
344: }
345: return(0);
346: }
351: PetscErrorCode PCRedundantGetPC_Redundant(PC pc,PC *innerpc)
352: {
353: PC_Redundant *red = (PC_Redundant*)pc->data;
356: *innerpc = red->pc;
357: return(0);
358: }
363: /*@
364: PCRedundantGetPC - Gets the sequential PC created by the redundant PC.
366: Not Collective
368: Input Parameter:
369: . pc - the preconditioner context
371: Output Parameter:
372: . innerpc - the sequential PC
374: Level: advanced
376: .keywords: PC, redundant solve
377: @*/
378: PetscErrorCode PCRedundantGetPC(PC pc,PC *innerpc)
379: {
380: PetscErrorCode ierr,(*f)(PC,PC*);
385: PetscObjectQueryFunction((PetscObject)pc,"PCRedundantGetPC_C",(void (**)(void))&f);
386: if (f) {
387: (*f)(pc,innerpc);
388: }
389: return(0);
390: }
395: PetscErrorCode PCRedundantGetOperators_Redundant(PC pc,Mat *mat,Mat *pmat)
396: {
397: PC_Redundant *red = (PC_Redundant*)pc->data;
400: if (mat) *mat = red->pmats;
401: if (pmat) *pmat = red->pmats;
402: return(0);
403: }
408: /*@
409: PCRedundantGetOperators - gets the sequential matrix and preconditioner matrix
411: Not Collective
413: Input Parameter:
414: . pc - the preconditioner context
416: Output Parameters:
417: + mat - the matrix
418: - pmat - the (possibly different) preconditioner matrix
420: Level: advanced
422: .keywords: PC, redundant solve
423: @*/
424: PetscErrorCode PCRedundantGetOperators(PC pc,Mat *mat,Mat *pmat)
425: {
426: PetscErrorCode ierr,(*f)(PC,Mat*,Mat*);
432: PetscObjectQueryFunction((PetscObject)pc,"PCRedundantGetOperators_C",(void (**)(void))&f);
433: if (f) {
434: (*f)(pc,mat,pmat);
435: }
436: return(0);
437: }
439: /* -------------------------------------------------------------------------------------*/
440: /*MC
441: PCREDUNDANT - Runs a preconditioner for the entire problem on subgroups of processors
443: Options for the redundant preconditioners can be set with -redundant_pc_xxx
445: Options Database:
446: . -pc_redundant_number <n> - number of redundant solves, for example if you are using 64 MPI processes and
447: use an n of 4 there will be 4 parallel solves each on 16 = 64/4 processes.
449: Level: intermediate
451: .seealso: PCCreate(), PCSetType(), PCType (for list of available types), PCRedundantSetScatter(),
452: PCRedundantGetPC(), PCRedundantGetOperators(), PCRedundantSetNumber()
453: M*/
458: PetscErrorCode PCCreate_Redundant(PC pc)
459: {
461: PC_Redundant *red;
462: PetscMPIInt size;
463:
465: PetscNew(PC_Redundant,&red);
466: PetscLogObjectMemory(pc,sizeof(PC_Redundant));
467: MPI_Comm_size(pc->comm,&size);
468: red->nsubcomm = size;
469: red->useparallelmat = PETSC_TRUE;
470: pc->data = (void*)red;
472: pc->ops->apply = PCApply_Redundant;
473: pc->ops->applytranspose = 0;
474: pc->ops->setup = PCSetUp_Redundant;
475: pc->ops->destroy = PCDestroy_Redundant;
476: pc->ops->setfromoptions = PCSetFromOptions_Redundant;
477: pc->ops->view = PCView_Redundant;
478: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCRedundantSetScatter_C","PCRedundantSetScatter_Redundant",
479: PCRedundantSetScatter_Redundant);
480: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCRedundantSetNumber_C","PCRedundantSetNumber_Redundant",
481: PCRedundantSetNumber_Redundant);
482: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCRedundantGetPC_C","PCRedundantGetPC_Redundant",
483: PCRedundantGetPC_Redundant);
484: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCRedundantGetOperators_C","PCRedundantGetOperators_Redundant",
485: PCRedundantGetOperators_Redundant);
486: return(0);
487: }