Actual source code: damgsnes.c

  1: #define PETSCSNES_DLL
  2: 
 3:  #include petscda.h
 4:  #include petscmg.h
 5:  #include petscdmmg.h

  7: #if defined(PETSC_HAVE_ADIC)
 14: #endif

 17: EXTERN PetscErrorCode  NLFCreate_DAAD(NLF*);
 18: EXTERN PetscErrorCode  NLFDAADSetDA_DAAD(NLF,DA);
 19: EXTERN PetscErrorCode  NLFDAADSetCtx_DAAD(NLF,void*);
 20: EXTERN PetscErrorCode  NLFDAADSetResidual_DAAD(NLF,Vec);
 21: EXTERN PetscErrorCode  NLFDAADSetNewtonIterations_DAAD(NLF,PetscInt);

 24: /*
 25:       period of -1 indicates update only on zeroth iteration of SNES
 26: */
 27: #define ShouldUpdate(l,it) (((dmmg[l-1]->updatejacobianperiod == -1) && (it == 0)) || \
 28:                             ((dmmg[l-1]->updatejacobianperiod >   0) && !(it % dmmg[l-1]->updatejacobianperiod)))
 29: /*
 30:    Evaluates the Jacobian on all of the grids. It is used by DMMG to provide the 
 31:    ComputeJacobian() function that SNESSetJacobian() requires.
 32: */
 35: PetscErrorCode DMMGComputeJacobian_Multigrid(SNES snes,Vec X,Mat *J,Mat *B,MatStructure *flag,void *ptr)
 36: {
 37:   DMMG           *dmmg = (DMMG*)ptr;
 39:   PetscInt       i,nlevels = dmmg[0]->nlevels,it;
 40:   KSP            ksp,lksp;
 41:   PC             pc;
 42:   PetscTruth     ismg;
 43:   Vec            W;
 44:   MatStructure   flg;

 47:   if (!dmmg) SETERRQ(PETSC_ERR_ARG_NULL,"Passing null as user context which should contain DMMG");
 48:   SNESGetIterationNumber(snes,&it);

 50:   /* compute Jacobian on finest grid */
 51:   if (dmmg[nlevels-1]->updatejacobian && ShouldUpdate(nlevels,it)) {
 52:     (*DMMGGetFine(dmmg)->computejacobian)(snes,X,J,B,flag,DMMGGetFine(dmmg));
 53:   } else {
 54:     PetscInfo3(0,"Skipping Jacobian, SNES iteration %D frequence %D level %D\n",it,dmmg[nlevels-1]->updatejacobianperiod,nlevels-1);
 55:     *flag = SAME_PRECONDITIONER;
 56:   }
 57:   MatSNESMFSetBase(DMMGGetFine(dmmg)->J,X);

 59:   /* create coarser grid Jacobians for preconditioner if multigrid is the preconditioner */
 60:   SNESGetKSP(snes,&ksp);
 61:   KSPGetPC(ksp,&pc);
 62:   PetscTypeCompare((PetscObject)pc,PCMG,&ismg);
 63:   if (ismg) {
 64:     PetscTruth galerkin;

 66:     PCMGGetGalerkin(pc,&galerkin);
 67:     PCMGGetSmoother(pc,nlevels-1,&lksp);
 68:     KSPSetOperators(lksp,DMMGGetFine(dmmg)->J,DMMGGetFine(dmmg)->B,*flag);

 70:     if (!galerkin) {
 71:       for (i=nlevels-1; i>0; i--) {
 72:         if (!dmmg[i-1]->w) {
 73:           VecDuplicate(dmmg[i-1]->x,&dmmg[i-1]->w);
 74:         }
 75:         W    = dmmg[i-1]->w;
 76:         /* restrict X to coarser grid */
 77:         MatRestrict(dmmg[i]->R,X,W);
 78:         X    = W;
 79:         /* scale to "natural" scaling for that grid */
 80:         VecPointwiseMult(X,X,dmmg[i]->Rscale);
 81:         /* tell the base vector for matrix free multiplies */
 82:         MatSNESMFSetBase(dmmg[i-1]->J,X);
 83:         /* compute Jacobian on coarse grid */
 84:         if (dmmg[i-1]->updatejacobian && ShouldUpdate(i,it)) {
 85:           (*dmmg[i-1]->computejacobian)(snes,X,&dmmg[i-1]->J,&dmmg[i-1]->B,&flg,dmmg[i-1]);
 86:           flg = SAME_NONZERO_PATTERN;
 87:         } else {
 88:           PetscInfo3(0,"Skipping Jacobian, SNES iteration %D frequence %D level %D\n",it,dmmg[i-1]->updatejacobianperiod,i-1);
 89:           flg = SAME_PRECONDITIONER;
 90:         }
 91:         PCMGGetSmoother(pc,i-1,&lksp);
 92:         KSPSetOperators(lksp,dmmg[i-1]->J,dmmg[i-1]->B,flg);
 93:       }
 94:     }
 95:   }
 96:   return(0);
 97: }

 99: /* ---------------------------------------------------------------------------*/


104: /* 
105:    DMMGFormFunction - This is a universal global FormFunction used by the DMMG code
106:    when the user provides a local function.

108:    Input Parameters:
109: +  snes - the SNES context
110: .  X - input vector
111: -  ptr - optional user-defined context, as set by SNESSetFunction()

113:    Output Parameter:
114: .  F - function vector

116:  */
117: PetscErrorCode DMMGFormFunction(SNES snes,Vec X,Vec F,void *ptr)
118: {
119:   DMMG           dmmg = (DMMG)ptr;
121:   Vec            localX;
122:   DA             da = (DA)dmmg->dm;

125:   DAGetLocalVector(da,&localX);
126:   /*
127:      Scatter ghost points to local vector, using the 2-step process
128:         DAGlobalToLocalBegin(), DAGlobalToLocalEnd().
129:   */
130:   DAGlobalToLocalBegin(da,X,INSERT_VALUES,localX);
131:   DAGlobalToLocalEnd(da,X,INSERT_VALUES,localX);
132:   DAFormFunction1(da,localX,F,dmmg->user);
133:   DARestoreLocalVector(da,&localX);
134:   return(0);
135: }

139: /* 
140:    DMMGFormFunctionFD - This is a universal global FormFunction used by the DMMG code
141:    when the user provides a local function used to compute the Jacobian via FD.

143:    Input Parameters:
144: +  snes - the SNES context
145: .  X - input vector
146: -  ptr - optional user-defined context, as set by SNESSetFunction()

148:    Output Parameter:
149: .  F - function vector

151:  */
152: PetscErrorCode DMMGFormFunctionFD(SNES snes,Vec X,Vec F,void *ptr)
153: {
154:   DMMG           dmmg = (DMMG)ptr;
156:   Vec            localX;
157:   DA             da = (DA)dmmg->dm;
158:   PetscInt       N,n;
159: 
161:   /* determine whether X=localX */
162:   DAGetLocalVector(da,&localX);
163:   VecGetSize(X,&N);
164:   VecGetSize(localX,&n);

166:   if (n != N){ /* X != localX */
167:     /* Scatter ghost points to local vector, using the 2-step process
168:        DAGlobalToLocalBegin(), DAGlobalToLocalEnd().
169:     */
170:     DAGlobalToLocalBegin(da,X,INSERT_VALUES,localX);
171:     DAGlobalToLocalEnd(da,X,INSERT_VALUES,localX);
172:   } else {
173:     DARestoreLocalVector(da,&localX);
174:     localX = X;
175:   }
176:   DAFormFunction(da,dmmg->lfj,localX,F,dmmg->user);
177:   if (n != N){
178:     DARestoreLocalVector(da,&localX);
179:   }
180:   return(0);
181: }

185: /*@C 
186:    SNESDAFormFunction - This is a universal function evaluation routine that
187:    may be used with SNESSetFunction() as long as the user context has a DA
188:    as its first record and the user has called DASetLocalFunction().

190:    Collective on SNES

192:    Input Parameters:
193: +  snes - the SNES context
194: .  X - input vector
195: .  F - function vector
196: -  ptr - pointer to a structure that must have a DA as its first entry. For example this 
197:          could be a DMMG, this ptr must have been passed into SNESDAFormFunction() as the context

199:    Level: intermediate

201: .seealso: DASetLocalFunction(), DASetLocalJacobian(), DASetLocalAdicFunction(), DASetLocalAdicMFFunction(),
202:           SNESSetFunction(), SNESSetJacobian()

204: @*/
205: PetscErrorCode  SNESDAFormFunction(SNES snes,Vec X,Vec F,void *ptr)
206: {
208:   Vec            localX;
209:   DA             da = *(DA*)ptr;
210:   PetscInt       N,n;
211: 
213:   if (!da) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Looks like you called SNESSetFromFuntion(snes,SNESDAFormFunction,) without the DA context");

215:   /* determine whether X=localX */
216:   DAGetLocalVector(da,&localX);
217:   VecGetSize(X,&N);
218:   VecGetSize(localX,&n);
219: 
220: 
221:   if (n != N){ /* X != localX */
222:     /* Scatter ghost points to local vector, using the 2-step process
223:         DAGlobalToLocalBegin(), DAGlobalToLocalEnd().
224:     */
225:     DAGlobalToLocalBegin(da,X,INSERT_VALUES,localX);
226:     DAGlobalToLocalEnd(da,X,INSERT_VALUES,localX);
227:   } else {
228:     DARestoreLocalVector(da,&localX);
229:     localX = X;
230:   }
231:   DAFormFunction1(da,localX,F,ptr);
232:   if (n != N){
233:     if (PetscExceptionValue(ierr)) {
234:       PetscErrorCode pDARestoreLocalVector(da,&localX);CHKERRQ(pierr);
235:     }
236: 
237:     DARestoreLocalVector(da,&localX);
238:   }
239:   return(0);
240: }

242: /* ------------------------------------------------------------------------------*/
243:  #include src/mat/matimpl.h
246: PetscErrorCode DMMGComputeJacobianWithFD(SNES snes,Vec x1,Mat *J,Mat *B,MatStructure *flag,void *ctx)
247: {
249:   DMMG           dmmg = (DMMG)ctx;
250:   MatFDColoring  color = (MatFDColoring)dmmg->fdcoloring;
251: 
253:   if (color->ctype == IS_COLORING_GHOSTED){
254:     DA            da=(DA)dmmg->dm;
255:     Vec           x1_loc;
256:     DAGetLocalVector(da,&x1_loc);
257:     DAGlobalToLocalBegin(da,x1,INSERT_VALUES,x1_loc);
258:     DAGlobalToLocalEnd(da,x1,INSERT_VALUES,x1_loc);
259:     SNESDefaultComputeJacobianColor(snes,x1_loc,J,B,flag,dmmg->fdcoloring);
260:     DARestoreLocalVector(da,&x1_loc);
261:   } else {
262:     SNESDefaultComputeJacobianColor(snes,x1,J,B,flag,dmmg->fdcoloring);
263:   }
264:   return(0);
265: }

269: PetscErrorCode DMMGComputeJacobianWithMF(SNES snes,Vec x1,Mat *J,Mat *B,MatStructure *flag,void *ctx)
270: {
272: 
274:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
275:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
276:   return(0);
277: }

281: /*
282:     DMMGComputeJacobian - Evaluates the Jacobian when the user has provided
283:     a local function evaluation routine.
284: */
285: PetscErrorCode DMMGComputeJacobian(SNES snes,Vec X,Mat *J,Mat *B,MatStructure *flag,void *ptr)
286: {
287:   DMMG           dmmg = (DMMG) ptr;
289:   Vec            localX;
290:   DA             da = (DA) dmmg->dm;

293:   DAGetLocalVector(da,&localX);
294:   DAGlobalToLocalBegin(da,X,INSERT_VALUES,localX);
295:   DAGlobalToLocalEnd(da,X,INSERT_VALUES,localX);
296:   DAComputeJacobian1(da,localX,*B,dmmg->user);
297:   DARestoreLocalVector(da,&localX);
298:   /* Assemble true Jacobian; if it is different */
299:   if (*J != *B) {
300:     MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
301:     MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
302:   }
303:   MatSetOption(*B,MAT_NEW_NONZERO_LOCATION_ERR);
304:   *flag = SAME_NONZERO_PATTERN;
305:   return(0);
306: }

310: /*
311:     SNESDAComputeJacobianWithAdifor - This is a universal Jacobian evaluation routine
312:     that may be used with SNESSetJacobian() from Fortran as long as the user context has 
313:     a DA as its first record and DASetLocalAdiforFunction() has been called.  

315:    Collective on SNES

317:    Input Parameters:
318: +  snes - the SNES context
319: .  X - input vector
320: .  J - Jacobian
321: .  B - Jacobian used in preconditioner (usally same as J)
322: .  flag - indicates if the matrix changed its structure
323: -  ptr - optional user-defined context, as set by SNESSetFunction()

325:    Level: intermediate

327: .seealso: DASetLocalFunction(), DASetLocalAdicFunction(), SNESSetFunction(), SNESSetJacobian()

329: */
330: PetscErrorCode  SNESDAComputeJacobianWithAdifor(SNES snes,Vec X,Mat *J,Mat *B,MatStructure *flag,void *ptr)
331: {
332:   DA             da = *(DA*) ptr;
334:   Vec            localX;

337:   DAGetLocalVector(da,&localX);
338:   DAGlobalToLocalBegin(da,X,INSERT_VALUES,localX);
339:   DAGlobalToLocalEnd(da,X,INSERT_VALUES,localX);
340:   DAComputeJacobian1WithAdifor(da,localX,*B,ptr);
341:   DARestoreLocalVector(da,&localX);
342:   /* Assemble true Jacobian; if it is different */
343:   if (*J != *B) {
344:     MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
345:     MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
346:   }
347:   MatSetOption(*B,MAT_NEW_NONZERO_LOCATION_ERR);
348:   *flag = SAME_NONZERO_PATTERN;
349:   return(0);
350: }

354: /*
355:    SNESDAComputeJacobian - This is a universal Jacobian evaluation routine for a
356:    locally provided Jacobian.

358:    Collective on SNES

360:    Input Parameters:
361: +  snes - the SNES context
362: .  X - input vector
363: .  J - Jacobian
364: .  B - Jacobian used in preconditioner (usally same as J)
365: .  flag - indicates if the matrix changed its structure
366: -  ptr - optional user-defined context, as set by SNESSetFunction()

368:    Level: intermediate

370: .seealso: DASetLocalFunction(), DASetLocalJacobian(), SNESSetFunction(), SNESSetJacobian()

372: */
373: PetscErrorCode  SNESDAComputeJacobian(SNES snes,Vec X,Mat *J,Mat *B,MatStructure *flag,void *ptr)
374: {
375:   DA             da = *(DA*) ptr;
377:   Vec            localX;

380:   DAGetLocalVector(da,&localX);
381:   DAGlobalToLocalBegin(da,X,INSERT_VALUES,localX);
382:   DAGlobalToLocalEnd(da,X,INSERT_VALUES,localX);
383:   DAComputeJacobian1(da,localX,*B,ptr);
384:   DARestoreLocalVector(da,&localX);
385:   /* Assemble true Jacobian; if it is different */
386:   if (*J != *B) {
387:     MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
388:     MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
389:   }
390:   MatSetOption(*B,MAT_NEW_NONZERO_LOCATION_ERR);
391:   *flag = SAME_NONZERO_PATTERN;
392:   return(0);
393: }

397: PetscErrorCode DMMGSolveSNES(DMMG *dmmg,PetscInt level)
398: {
400:   PetscInt       nlevels = dmmg[0]->nlevels;

403:   dmmg[0]->nlevels = level+1;
404:   SNESSolve(dmmg[level]->snes,PETSC_NULL,dmmg[level]->x);
405:   dmmg[0]->nlevels = nlevels;
406:   return(0);
407: }

409: /* ===========================================================================================================*/

413: /*@C
414:     DMMGSetSNES - Sets the nonlinear function that defines the nonlinear set of equations
415:     to be solved using the grid hierarchy.

417:     Collective on DMMG

419:     Input Parameter:
420: +   dmmg - the context
421: .   function - the function that defines the nonlinear system
422: -   jacobian - optional function to compute Jacobian

424:     Options Database Keys:
425: +    -dmmg_snes_monitor
426: .    -dmmg_jacobian_fd
427: .    -dmmg_jacobian_ad
428: .    -dmmg_jacobian_mf_fd_operator
429: .    -dmmg_jacobian_mf_fd
430: .    -dmmg_jacobian_mf_ad_operator
431: .    -dmmg_jacobian_mf_ad
432: .    -dmmg_iscoloring_type
433: -    -dmmg_jacobian_period <p> - Indicates how often in the SNES solve the Jacobian is recomputed (on all levels)
434:                                  as suggested by Florin Dobrian if p is -1 then Jacobian is computed only on first
435:                                  SNES iteration (i.e. -1 is equivalent to infinity) 

437:     Level: advanced

439: .seealso DMMGCreate(), DMMGDestroy, DMMGSetKSP(), DMMGSetSNESLocal()

441: @*/
442: PetscErrorCode  DMMGSetSNES(DMMG *dmmg,PetscErrorCode (*function)(SNES,Vec,Vec,void*),PetscErrorCode (*jacobian)(SNES,Vec,Mat*,Mat*,MatStructure*,void*))
443: {
445:   PetscInt       i,nlevels = dmmg[0]->nlevels,period = 1,isctype=0;
446:   PetscTruth     snesmonitor,mffdoperator,mffd,fdjacobian;
447: #if defined(PETSC_HAVE_ADIC)
448:   PetscTruth     mfadoperator,mfad,adjacobian;
449: #endif
450:   PetscViewer    ascii;
451:   MPI_Comm       comm;
452:   PetscMPIInt    size;
453:   const char     *isctypes[] = {"IS_COLORING_LOCAL","IS_COLORING_GHOSTED"};
454:   ISColoringType ctype;

457:   if (!dmmg)     SETERRQ(PETSC_ERR_ARG_NULL,"Passing null as DMMG");
458:   if (!jacobian) jacobian = DMMGComputeJacobianWithFD;

460:   PetscOptionsBegin(dmmg[0]->comm,PETSC_NULL,"DMMG Options","SNES");
461:     PetscOptionsName("-dmmg_snes_monitor","Monitor nonlinear convergence","SNESSetMonitor",&snesmonitor);


464:     PetscOptionsName("-dmmg_jacobian_fd","Compute sparse Jacobian explicitly with finite differencing","DMMGSetSNES",&fdjacobian);
465:     if (fdjacobian) jacobian = DMMGComputeJacobianWithFD;
466: #if defined(PETSC_HAVE_ADIC)
467:     PetscOptionsName("-dmmg_jacobian_ad","Compute sparse Jacobian explicitly with ADIC (automatic differentiation)","DMMGSetSNES",&adjacobian);
468:     if (adjacobian) jacobian = DMMGComputeJacobianWithAdic;
469: #endif

471:     PetscOptionsTruthGroupBegin("-dmmg_jacobian_mf_fd_operator","Apply Jacobian via matrix free finite differencing","DMMGSetSNES",&mffdoperator);
472:     PetscOptionsTruthGroupEnd("-dmmg_jacobian_mf_fd","Apply Jacobian via matrix free finite differencing even in computing preconditioner","DMMGSetSNES",&mffd);
473:     if (mffd) mffdoperator = PETSC_TRUE;
474: #if defined(PETSC_HAVE_ADIC)
475:     PetscOptionsTruthGroupBegin("-dmmg_jacobian_mf_ad_operator","Apply Jacobian via matrix free ADIC (automatic differentiation)","DMMGSetSNES",&mfadoperator);
476:     PetscOptionsTruthGroupEnd("-dmmg_jacobian_mf_ad","Apply Jacobian via matrix free ADIC (automatic differentiation) even in computing preconditioner","DMMGSetSNES",&mfad);
477:     if (mfad) mfadoperator = PETSC_TRUE;
478: #endif
479:     MPI_Comm_size(dmmg[0]->comm,&size);
480:     if (size == 1){
481:       isctype = 0;
482:     } else {
483:       isctype = 1;
484:     }
485:     PetscOptionsEList("-dmmg_iscoloring_type","Type of ISColoring","None",isctypes,2,isctypes[isctype],&isctype,PETSC_NULL);
486: 
487:   PetscOptionsEnd();

489:   /* create solvers for each level */
490:   for (i=0; i<nlevels; i++) {
491:     SNESCreate(dmmg[i]->comm,&dmmg[i]->snes);
492:     SNESGetKSP(dmmg[i]->snes,&dmmg[i]->ksp);
493:     if (snesmonitor) {
494:       PetscObjectGetComm((PetscObject)dmmg[i]->snes,&comm);
495:       PetscViewerASCIIOpen(comm,"stdout",&ascii);
496:       PetscViewerASCIISetTab(ascii,nlevels-i);
497:       SNESSetMonitor(dmmg[i]->snes,SNESDefaultMonitor,ascii,(PetscErrorCode(*)(void*))PetscViewerDestroy);
498:     }

500:     if (mffdoperator) {
501:       MatCreateSNESMF(dmmg[i]->snes,dmmg[i]->x,&dmmg[i]->J);
502:       VecDuplicate(dmmg[i]->x,&dmmg[i]->work1);
503:       VecDuplicate(dmmg[i]->x,&dmmg[i]->work2);
504:       MatSNESMFSetFunction(dmmg[i]->J,dmmg[i]->work1,function,dmmg[i]);
505:       if (mffd) {
506:         dmmg[i]->B = dmmg[i]->J;
507:         jacobian   = DMMGComputeJacobianWithMF;
508:       }
509: #if defined(PETSC_HAVE_ADIC)
510:     } else if (mfadoperator) {
511:       MatRegisterDAAD();
512:       MatCreateDAAD((DA)dmmg[i]->dm,&dmmg[i]->J);
513:       MatDAADSetCtx(dmmg[i]->J,dmmg[i]->user);
514:       if (mfad) {
515:         dmmg[i]->B = dmmg[i]->J;
516:         jacobian   = DMMGComputeJacobianWithMF;
517:       }
518: #endif
519:     }
520: 
521:     if (!dmmg[i]->B) {
522:       DMGetMatrix(dmmg[i]->dm,MATAIJ,&dmmg[i]->B);
523:     }
524:     if (!dmmg[i]->J) {
525:       dmmg[i]->J = dmmg[i]->B;
526:     }

528:     DMMGSetUpLevel(dmmg,dmmg[i]->ksp,i+1);
529: 
530:     /*
531:        if the number of levels is > 1 then we want the coarse solve in the grid sequencing to use LU
532:        when possible 
533:     */
534:     if (nlevels > 1 && i == 0) {
535:       PC         pc;
536:       KSP        cksp;
537:       PetscTruth flg1,flg2,flg3;

539:       KSPGetPC(dmmg[i]->ksp,&pc);
540:       PCMGGetCoarseSolve(pc,&cksp);
541:       KSPGetPC(cksp,&pc);
542:       PetscTypeCompare((PetscObject)pc,PCILU,&flg1);
543:       PetscTypeCompare((PetscObject)pc,PCSOR,&flg2);
544:       PetscTypeCompare((PetscObject)pc,PETSC_NULL,&flg3);
545:       if (flg1 || flg2 || flg3) {
546:         PCSetType(pc,PCLU);
547:       }
548:     }

550:     dmmg[i]->solve           = DMMGSolveSNES;
551:     dmmg[i]->computejacobian = jacobian;
552:     dmmg[i]->computefunction = function;
553:   }

555:   if (jacobian == DMMGComputeJacobianWithFD) {
556:     ISColoring iscoloring;
557:     ctype = (ISColoringType)isctype;

559:     for (i=0; i<nlevels; i++) {
560:       DMGetColoring(dmmg[i]->dm,ctype,&iscoloring);
561:       MatFDColoringCreate(dmmg[i]->B,iscoloring,&dmmg[i]->fdcoloring);
562:       ISColoringDestroy(iscoloring);
563:       if (function == DMMGFormFunction) function = DMMGFormFunctionFD;
564:       MatFDColoringSetFunction(dmmg[i]->fdcoloring,(PetscErrorCode(*)(void))function,dmmg[i]);
565:       MatFDColoringSetFromOptions(dmmg[i]->fdcoloring);
566:     }
567: #if defined(PETSC_HAVE_ADIC)
568:   } else if (jacobian == DMMGComputeJacobianWithAdic) {
569:     for (i=0; i<nlevels; i++) {
570:       ISColoring iscoloring;
571:       DMGetColoring(dmmg[i]->dm,IS_COLORING_GHOSTED,&iscoloring);
572:       MatSetColoring(dmmg[i]->B,iscoloring);
573:       ISColoringDestroy(iscoloring);
574:     }
575: #endif
576:   }
577:   for (i=0; i<nlevels; i++) {
578:     SNESSetJacobian(dmmg[i]->snes,dmmg[i]->J,dmmg[i]->B,DMMGComputeJacobian_Multigrid,dmmg);
579:     SNESSetFunction(dmmg[i]->snes,dmmg[i]->b,function,dmmg[i]);
580:     SNESSetFromOptions(dmmg[i]->snes);
581:   }

583:   /* Create interpolation scaling */
584:   for (i=1; i<nlevels; i++) {
585:     DMGetInterpolationScale(dmmg[i-1]->dm,dmmg[i]->dm,dmmg[i]->R,&dmmg[i]->Rscale);
586:   }

588:   PetscOptionsGetInt(PETSC_NULL,"-dmmg_jacobian_period",&period,PETSC_NULL);
589:   for (i=0; i<nlevels; i++) {
590:     dmmg[i]->updatejacobian       = PETSC_TRUE;
591:     dmmg[i]->updatejacobianperiod = period;
592:   }

594: #if defined(PETSC_HAVE_ADIC)
595:   {
596:     PetscTruth flg;
597:     PetscOptionsHasName(PETSC_NULL,"-dmmg_fas",&flg);
598:     if (flg) {
599:       PetscTruth block = PETSC_FALSE;
600:       PetscTruth ngmres = PETSC_FALSE;
601:       PetscInt   newton_its;
602:       PetscOptionsHasName(0,"-dmmg_fas_view",&flg);
603:       for (i=0; i<nlevels; i++) {
604:         NLFCreate_DAAD(&dmmg[i]->nlf);
605:         NLFDAADSetDA_DAAD(dmmg[i]->nlf,(DA)dmmg[i]->dm);
606:         NLFDAADSetCtx_DAAD(dmmg[i]->nlf,dmmg[i]->user);
607:         NLFDAADSetResidual_DAAD(dmmg[i]->nlf,dmmg[i]->r);
608:         VecDuplicate(dmmg[i]->b,&dmmg[i]->w);

610:         dmmg[i]->monitor    = PETSC_FALSE;
611:         PetscOptionsHasName(0,"-dmmg_fas_monitor",&dmmg[i]->monitor);
612:         dmmg[i]->monitorall = PETSC_FALSE;
613:         PetscOptionsHasName(0,"-dmmg_fas_monitor_all",&dmmg[i]->monitorall);
614:         dmmg[i]->presmooth  = 2;
615:         PetscOptionsGetInt(0,"-dmmg_fas_presmooth",&dmmg[i]->presmooth,0);
616:         dmmg[i]->postsmooth = 2;
617:         PetscOptionsGetInt(0,"-dmmg_fas_postsmooth",&dmmg[i]->postsmooth,0);
618:         dmmg[i]->coarsesmooth = 2;
619:         PetscOptionsGetInt(0,"-dmmg_fas_coarsesmooth",&dmmg[i]->coarsesmooth,0);

621:         dmmg[i]->rtol = 1.e-8;
622:         PetscOptionsGetReal(0,"-dmmg_fas_rtol",&dmmg[i]->rtol,0);
623:         dmmg[i]->abstol = 1.e-50;
624:         PetscOptionsGetReal(0,"-dmmg_fas_atol",&dmmg[i]->abstol,0);

626:         newton_its = 2;
627:         PetscOptionsGetInt(0,"-dmmg_fas_newton_its",&newton_its,0);
628:         NLFDAADSetNewtonIterations_DAAD(dmmg[i]->nlf,newton_its);

630:         if (flg) {
631:           if (i == 0) {
632:             PetscPrintf(dmmg[i]->comm,"FAS Solver Parameters\n");
633:             PetscPrintf(dmmg[i]->comm,"  rtol %G atol %G\n",dmmg[i]->rtol,dmmg[i]->abstol);
634:             PetscPrintf(dmmg[i]->comm,"             coarsesmooths %D\n",dmmg[i]->coarsesmooth);
635:             PetscPrintf(dmmg[i]->comm,"             Newton iterations %D\n",newton_its);
636:           } else {
637:             PetscPrintf(dmmg[i]->comm,"  level %D   presmooths    %D\n",i,dmmg[i]->presmooth);
638:             PetscPrintf(dmmg[i]->comm,"             postsmooths   %D\n",dmmg[i]->postsmooth);
639:             PetscPrintf(dmmg[i]->comm,"             Newton iterations %D\n",newton_its);
640:           }
641:         }
642:         PetscOptionsHasName(0,"-dmmg_fas_block",&block);
643:         PetscOptionsHasName(0,"-dmmg_fas_ngmres",&ngmres);
644:         if (block) {
645:           dmmg[i]->solve = DMMGSolveFASb;
646:           if (flg) {
647:             PetscPrintf(dmmg[i]->comm,"  using point-block smoothing\n");
648:           }
649:         } else if(ngmres) {
650:           dmmg[i]->solve = DMMGSolveFAS_NGMRES;
651:           if (flg) {
652:             PetscPrintf(dmmg[i]->comm,"  using non-linear gmres\n");
653:           }
654:         } else {
655:           dmmg[i]->solve = DMMGSolveFAS4;
656:         }
657:       }
658:     }
659:   }
660: #endif
661: 
662:   return(0);
663: }

667: /*@
668:     DMMGSetSNESLocalFD - Sets the local user function that is used to approximately compute the Jacobian
669:         via finite differences.

671:     Collective on DMMG

673:     Input Parameter:
674: +   dmmg - the context
675: -   function - the function that defines the nonlinear system

677:     Level: intermediate

679: .seealso DMMGCreate(), DMMGDestroy, DMMGSetKSP(), DMMGSetSNES(), DMMGSetSNESLocal()

681: @*/
682: PetscErrorCode DMMGSetSNESLocalFD(DMMG *dmmg,DALocalFunction1 function)
683: {
684:   PetscInt       i,nlevels = dmmg[0]->nlevels;

687:   for (i=0; i<nlevels; i++) {
688:     dmmg[i]->lfj = (PetscErrorCode (*)(void))function;
689:   }
690:   return(0);
691: }


694: /*M
695:     DMMGSetSNESLocal - Sets the local user function that defines the nonlinear set of equations
696:     that will use the grid hierarchy and (optionally) its derivative.

698:     Collective on DMMG

700:    Synopsis:
701:    PetscErrorCode DMMGSetSNESLocal(DMMG *dmmg,DALocalFunction1 function, DALocalFunction1 jacobian,
702:                         DALocalFunction1 ad_function, DALocalFunction1 admf_function);

704:     Input Parameter:
705: +   dmmg - the context
706: .   function - the function that defines the nonlinear system
707: .   jacobian - function defines the local part of the Jacobian
708: .   ad_function - the name of the function with an ad_ prefix. This is ignored if ADIC is
709:                   not installed
710: -   admf_function - the name of the function with an ad_ prefix. This is ignored if ADIC is
711:                   not installed

713:     Options Database Keys:
714: +    -dmmg_snes_monitor
715: .    -dmmg_jacobian_fd
716: .    -dmmg_jacobian_ad
717: .    -dmmg_jacobian_mf_fd_operator
718: .    -dmmg_jacobian_mf_fd
719: .    -dmmg_jacobian_mf_ad_operator
720: .    -dmmg_jacobian_mf_ad
721: -    -dmmg_jacobian_period <p> - Indicates how often in the SNES solve the Jacobian is recomputed (on all levels)
722:                                  as suggested by Florin Dobrian if p is -1 then Jacobian is computed only on first
723:                                  SNES iteration (i.e. -1 is equivalent to infinity) 


726:     Level: intermediate

728:     Notes: 
729:     If ADIC or ADIFOR have been installed, this routine can use ADIC or ADIFOR to compute
730:     the derivative; however, that function cannot call other functions except those in
731:     standard C math libraries.

733:     If ADIC/ADIFOR have not been installed and the Jacobian is not provided, this routine
734:     uses finite differencing to approximate the Jacobian.

736: .seealso DMMGCreate(), DMMGDestroy, DMMGSetKSP(), DMMGSetSNES()

738: M*/

742: PetscErrorCode DMMGSetSNESLocal_Private(DMMG *dmmg,DALocalFunction1 function,DALocalFunction1 jacobian,DALocalFunction1 ad_function,DALocalFunction1 admf_function)
743: {
745:   PetscInt       i,nlevels = dmmg[0]->nlevels;
746:   PetscErrorCode (*computejacobian)(SNES,Vec,Mat*,Mat*,MatStructure*,void*) = 0;


750:   if (jacobian)         computejacobian = DMMGComputeJacobian;
751: #if defined(PETSC_HAVE_ADIC)
752:   else if (ad_function) computejacobian = DMMGComputeJacobianWithAdic;
753: #endif

755:   DMMGSetSNES(dmmg,DMMGFormFunction,computejacobian);
756:   for (i=0; i<nlevels; i++) {
757:     DASetLocalFunction((DA)dmmg[i]->dm,function);
758:     dmmg[i]->lfj = (PetscErrorCode (*)(void))function;
759:     DASetLocalJacobian((DA)dmmg[i]->dm,jacobian);
760:     DASetLocalAdicFunction((DA)dmmg[i]->dm,ad_function);
761:     DASetLocalAdicMFFunction((DA)dmmg[i]->dm,admf_function);
762:   }
763:   return(0);
764: }

768: PetscErrorCode DMMGFunctioni(PetscInt i,Vec u,PetscScalar* r,void* ctx)
769: {
770:   DMMG           dmmg = (DMMG)ctx;
771:   Vec            U = dmmg->lwork1;
773:   VecScatter     gtol;

776:   /* copy u into interior part of U */
777:   DAGetScatter((DA)dmmg->dm,0,&gtol,0);
778:   VecScatterBegin(u,U,INSERT_VALUES,SCATTER_FORWARD_LOCAL,gtol);
779:   VecScatterEnd(u,U,INSERT_VALUES,SCATTER_FORWARD_LOCAL,gtol);
780:   DAFormFunctioni1((DA)dmmg->dm,i,U,r,dmmg->user);
781:   return(0);
782: }

786: PetscErrorCode DMMGFunctionib(PetscInt i,Vec u,PetscScalar* r,void* ctx)
787: {
788:   DMMG           dmmg = (DMMG)ctx;
789:   Vec            U = dmmg->lwork1;
791:   VecScatter     gtol;

794:   /* copy u into interior part of U */
795:   DAGetScatter((DA)dmmg->dm,0,&gtol,0);
796:   VecScatterBegin(u,U,INSERT_VALUES,SCATTER_FORWARD_LOCAL,gtol);
797:   VecScatterEnd(u,U,INSERT_VALUES,SCATTER_FORWARD_LOCAL,gtol);
798:   DAFormFunctionib1((DA)dmmg->dm,i,U,r,dmmg->user);
799:   return(0);
800: }

804: PetscErrorCode DMMGFunctioniBase(Vec u,void* ctx)
805: {
806:   DMMG           dmmg = (DMMG)ctx;
807:   Vec            U = dmmg->lwork1;

811:   DAGlobalToLocalBegin((DA)dmmg->dm,u,INSERT_VALUES,U);
812:   DAGlobalToLocalEnd((DA)dmmg->dm,u,INSERT_VALUES,U);
813:   return(0);
814: }

818: PetscErrorCode DMMGSetSNESLocali_Private(DMMG *dmmg,PetscErrorCode (*functioni)(DALocalInfo*,MatStencil*,void*,PetscScalar*,void*),PetscErrorCode (*adi)(DALocalInfo*,MatStencil*,void*,void*,void*),PetscErrorCode (*adimf)(DALocalInfo*,MatStencil*,void*,void*,void*))
819: {
821:   PetscInt       i,nlevels = dmmg[0]->nlevels;

824:   for (i=0; i<nlevels; i++) {
825:     DASetLocalFunctioni((DA)dmmg[i]->dm,functioni);
826:     DASetLocalAdicFunctioni((DA)dmmg[i]->dm,adi);
827:     DASetLocalAdicMFFunctioni((DA)dmmg[i]->dm,adimf);
828:     MatSNESMFSetFunctioni(dmmg[i]->J,DMMGFunctioni);
829:     MatSNESMFSetFunctioniBase(dmmg[i]->J,DMMGFunctioniBase);
830:     if (!dmmg[i]->lwork1) {
831:       DACreateLocalVector((DA)dmmg[i]->dm,&dmmg[i]->lwork1);
832:     }
833:   }
834:   return(0);
835: }

839: PetscErrorCode DMMGSetSNESLocalib_Private(DMMG *dmmg,PetscErrorCode (*functioni)(DALocalInfo*,MatStencil*,void*,PetscScalar*,void*),PetscErrorCode (*adi)(DALocalInfo*,MatStencil*,void*,void*,void*),PetscErrorCode (*adimf)(DALocalInfo*,MatStencil*,void*,void*,void*))
840: {
842:   PetscInt       i,nlevels = dmmg[0]->nlevels;

845:   for (i=0; i<nlevels; i++) {
846:     DASetLocalFunctionib((DA)dmmg[i]->dm,functioni);
847:     DASetLocalAdicFunctionib((DA)dmmg[i]->dm,adi);
848:     DASetLocalAdicMFFunctionib((DA)dmmg[i]->dm,adimf);
849:     if (!dmmg[i]->lwork1) {
850:       DACreateLocalVector((DA)dmmg[i]->dm,&dmmg[i]->lwork1);
851:     }
852:   }
853:   return(0);
854: }

856: static PetscErrorCode (*localfunc)(DALocalInfo*,void*) = 0;

860: PetscErrorCode  DMMGInitialGuess_Local(DMMG dmmg,Vec x)
861: {
863:   DALocalInfo    info;
864:   void           *f;
865:   DA             da = (DA)dmmg->dm;

868:   DAGetLocalInfo(da,&info);
869:   DAVecGetArray(da,x,&f);

871:   CHKMEMQ;
872:   (*localfunc)(&info,f);
873:   CHKMEMQ;

875:   return(0);
876: }

880: /*@C
881:     DMMGSetInitialGuessLocal - sets code to compute the initial guess for each level

883:     Collective on DMMG

885:     Input Parameter:
886: +   dmmg - the context
887: -   localguess - the function that computes the initial guess

889:     Level: intermediate

891: .seealso DMMGCreate(), DMMGDestroy, DMMGSetKSP(), DMMGSetSNES(), DMMGSetInitialGuess(), DMMGSetSNESLocal()

893: @*/
894: PetscErrorCode DMMGSetInitialGuessLocal(DMMG *dmmg,PetscErrorCode (*localguess)(DALocalInfo*,void*))
895: {


900:   localfunc = localguess;  /* stash into ugly static for now */

902:   DMMGSetInitialGuess(dmmg,DMMGInitialGuess_Local);
903:   return(0);
904: }